From d8bbc7858622b6d9c278469aab701ca0b609cddf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 15 May 2024 05:35:49 +0200 Subject: Merging upstream version 126.0. Signed-off-by: Daniel Baumann --- js/src/jit/CodeGenerator.cpp | 552 +++++++++++++++++++++++++++++++------------ 1 file changed, 400 insertions(+), 152 deletions(-) (limited to 'js/src/jit/CodeGenerator.cpp') diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index 10a69f0cb3..559ac50cc7 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -4205,13 +4205,23 @@ void CodeGenerator::visitGuardShape(LGuardShape* guard) { } void CodeGenerator::visitGuardFuse(LGuardFuse* guard) { + auto fuseIndex = guard->mir()->fuseIndex(); + switch (fuseIndex) { + case RealmFuses::FuseIndex::OptimizeGetIteratorFuse: + addOptimizeGetIteratorFuseDependency(); + return; + default: + // validateAndRegisterFuseDependencies doesn't have + // handling for this yet, actively check fuse instead. + break; + } + Register temp = ToRegister(guard->temp0()); Label bail; // Bake specific fuse address for Ion code, because we won't share this code // across realms. - GuardFuse* fuse = - mirGen().realm->realmFuses().getFuseByIndex(guard->mir()->fuseIndex()); + GuardFuse* fuse = mirGen().realm->realmFuses().getFuseByIndex(fuseIndex); masm.loadPtr(AbsoluteAddress(fuse->fuseRef()), temp); masm.branchPtr(Assembler::NotEqual, temp, ImmPtr(nullptr), &bail); @@ -6269,7 +6279,8 @@ void CodeGenerator::visitCallKnown(LCallKnown* call) { UnusedStackBytesForCall(call->mir()->paddedNumStackArgs()); WrappedFunction* target = call->getSingleTarget(); - // Native single targets (except wasm) are handled by LCallNative. + // Native single targets (except Wasm and TrampolineNative functions) are + // handled by LCallNative. MOZ_ASSERT(target->hasJitEntry()); // Missing arguments must have been explicitly appended by WarpBuilder. @@ -6341,12 +6352,7 @@ void CodeGenerator::visitCallKnown(LCallKnown* call) { template void CodeGenerator::emitCallInvokeFunction(T* apply) { - Register objreg = ToRegister(apply->getTempObject()); - - // Push the space used by the arguments. - masm.moveStackPtrTo(objreg); - - pushArg(objreg); // argv. + pushArg(masm.getStackPointer()); // argv. pushArg(ToRegister(apply->getArgc())); // argc. pushArg(Imm32(apply->mir()->ignoresReturnValue())); // ignoresReturnValue. pushArg(Imm32(apply->mir()->isConstructing())); // isConstructing. @@ -6370,7 +6376,13 @@ void CodeGenerator::emitAllocateSpaceForApply(Register argcreg, "Stack padding assumes that the frameSize is correct"); MOZ_ASSERT(JitStackValueAlignment == 2); Label noPaddingNeeded; - // if the number of arguments is odd, then we do not need any padding. + // If the number of arguments is odd, then we do not need any padding. + // + // Note: The |JitStackValueAlignment == 2| condition requires that the + // overall number of values on the stack is even. When we have an odd number + // of arguments, we don't need any padding, because the |thisValue| is + // pushed after the arguments, so the overall number of values on the stack + // is even. masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded); masm.addPtr(Imm32(1), scratch); masm.bind(&noPaddingNeeded); @@ -6382,13 +6394,13 @@ void CodeGenerator::emitAllocateSpaceForApply(Register argcreg, masm.subFromStackPtr(scratch); #ifdef DEBUG - // Put a magic value in the space reserved for padding. Note, this code - // cannot be merged with the previous test, as not all architectures can - // write below their stack pointers. + // Put a magic value in the space reserved for padding. Note, this code cannot + // be merged with the previous test, as not all architectures can write below + // their stack pointers. if (JitStackValueAlignment > 1) { MOZ_ASSERT(JitStackValueAlignment == 2); Label noPaddingNeeded; - // if the number of arguments is odd, then we do not need any padding. + // If the number of arguments is odd, then we do not need any padding. masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded); BaseValueIndex dstPtr(masm.getStackPointer(), argcreg); masm.storeValue(MagicValue(JS_ARG_POISON), dstPtr); @@ -6403,8 +6415,8 @@ void CodeGenerator::emitAllocateSpaceForConstructAndPushNewTarget( Register argcreg, Register newTargetAndScratch) { // Align the JitFrameLayout on the JitStackAlignment. Contrary to // |emitAllocateSpaceForApply()|, we're always pushing a magic value, because - // we can't write to |newTargetAndScratch| before |new.target| has - // been pushed onto the stack. + // we can't write to |newTargetAndScratch| before |new.target| has been pushed + // onto the stack. if (JitStackValueAlignment > 1) { MOZ_ASSERT(frameSize() % JitStackAlignment == 0, "Stack padding assumes that the frameSize is correct"); @@ -6412,6 +6424,12 @@ void CodeGenerator::emitAllocateSpaceForConstructAndPushNewTarget( Label noPaddingNeeded; // If the number of arguments is even, then we do not need any padding. + // + // Note: The |JitStackValueAlignment == 2| condition requires that the + // overall number of values on the stack is even. When we have an even + // number of arguments, we don't need any padding, because |new.target| is + // is pushed before the arguments and |thisValue| is pushed after all + // arguments, so the overall number of values on the stack is even. masm.branchTestPtr(Assembler::Zero, argcreg, Imm32(1), &noPaddingNeeded); masm.pushValue(MagicValue(JS_ARG_POISON)); masm.bind(&noPaddingNeeded); @@ -6437,9 +6455,8 @@ void CodeGenerator::emitCopyValuesForApply(Register argvSrcBase, Label loop; masm.bind(&loop); - // As argvIndex is off by 1, and we use the decBranchPtr instruction - // to loop back, we have to substract the size of the word which are - // copied. + // As argvIndex is off by 1, and we use the decBranchPtr instruction to loop + // back, we have to substract the size of the word which are copied. BaseValueIndex srcPtr(argvSrcBase, argvIndex, int32_t(argvSrcOffset) - sizeof(void*)); BaseValueIndex dstPtr(masm.getStackPointer(), argvIndex, @@ -6488,6 +6505,9 @@ void CodeGenerator::emitPushArguments(Register argcreg, Register scratch, // clang-format on // Compute the source and destination offsets into the stack. + // + // The |extraFormals| parameter is used when copying rest-parameters and + // allows to skip the initial parameters before the actual rest-parameters. Register argvSrcBase = FramePointer; size_t argvSrcOffset = JitFrameLayout::offsetOfActualArgs() + extraFormals * sizeof(JS::Value); @@ -6500,17 +6520,18 @@ void CodeGenerator::emitPushArguments(Register argcreg, Register scratch, emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset, argvDstOffset); - // Join with all arguments copied and the extra stack usage computed. + // Join with all arguments copied. masm.bind(&end); } -void CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply, - Register scratch) { - // Holds the function nargs. Initially the number of args to the caller. +void CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply) { + // Holds the function nargs. Register argcreg = ToRegister(apply->getArgc()); Register copyreg = ToRegister(apply->getTempObject()); + Register scratch = ToRegister(apply->getTempForArgCopy()); uint32_t extraFormals = apply->numExtraFormals(); + // Allocate space on the stack for arguments. emitAllocateSpaceForApply(argcreg, scratch); emitPushArguments(argcreg, scratch, copyreg, extraFormals); @@ -6519,22 +6540,21 @@ void CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply, masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex)); } -void CodeGenerator::emitPushArguments(LApplyArgsObj* apply, Register scratch) { - // argc and argsObj are mapped to the same calltemp register. - MOZ_ASSERT(apply->getArgsObj() == apply->getArgc()); - - Register tmpArgc = ToRegister(apply->getTempObject()); +void CodeGenerator::emitPushArguments(LApplyArgsObj* apply) { Register argsObj = ToRegister(apply->getArgsObj()); + Register tmpArgc = ToRegister(apply->getTempObject()); + Register scratch = ToRegister(apply->getTempForArgCopy()); + + // argc and argsObj are mapped to the same calltemp register. + MOZ_ASSERT(argsObj == ToRegister(apply->getArgc())); // Load argc into tmpArgc. - Address lengthAddr(argsObj, ArgumentsObject::getInitialLengthSlotOffset()); - masm.unboxInt32(lengthAddr, tmpArgc); - masm.rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), tmpArgc); + masm.loadArgumentsObjectLength(argsObj, tmpArgc); - // Allocate space on the stack for arguments. This modifies scratch. + // Allocate space on the stack for arguments. emitAllocateSpaceForApply(tmpArgc, scratch); - // Load arguments data + // Load arguments data. masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), argsObj); size_t argsSrcOffset = ArgumentsData::offsetOfArgs(); @@ -6543,6 +6563,7 @@ void CodeGenerator::emitPushArguments(LApplyArgsObj* apply, Register scratch) { // After this call, the argsObj register holds the argument count instead. emitPushArrayAsArguments(tmpArgc, argsObj, scratch, argsSrcOffset); + // Push |this|. masm.pushValue(ToValue(apply, LApplyArgsObj::ThisIndex)); } @@ -6566,69 +6587,72 @@ void CodeGenerator::emitPushArrayAsArguments(Register tmpArgc, // Skip the copy of arguments if there are none. masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy); + { + // Copy the values. This code is skipped entirely if there are no values. + size_t argvDstOffset = 0; - // Copy the values. This code is skipped entirely if there are - // no values. - size_t argvDstOffset = 0; - - Register argvSrcBase = srcBaseAndArgc; - Register copyreg = scratch; - - masm.push(tmpArgc); - Register argvIndex = tmpArgc; - argvDstOffset += sizeof(void*); + Register argvSrcBase = srcBaseAndArgc; - // Copy - emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset, - argvDstOffset); + // Stash away |tmpArgc| and adjust argvDstOffset accordingly. + masm.push(tmpArgc); + Register argvIndex = tmpArgc; + argvDstOffset += sizeof(void*); - // Restore. - masm.pop(srcBaseAndArgc); // srcBaseAndArgc now contains argc. - masm.jump(&epilogue); + // Copy + emitCopyValuesForApply(argvSrcBase, argvIndex, scratch, argvSrcOffset, + argvDstOffset); - // Clear argc if we skipped the copy step. + // Restore. + masm.pop(srcBaseAndArgc); // srcBaseAndArgc now contains argc. + masm.jump(&epilogue); + } masm.bind(&noCopy); - masm.movePtr(ImmWord(0), srcBaseAndArgc); + { + // Clear argc if we skipped the copy step. + masm.movePtr(ImmWord(0), srcBaseAndArgc); + } - // Join with all arguments copied and the extra stack usage computed. + // Join with all arguments copied. // Note, "srcBase" has become "argc". masm.bind(&epilogue); } -void CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply, - Register scratch) { +void CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply) { + Register elements = ToRegister(apply->getElements()); Register tmpArgc = ToRegister(apply->getTempObject()); - Register elementsAndArgc = ToRegister(apply->getElements()); + Register scratch = ToRegister(apply->getTempForArgCopy()); + + // argc and elements are mapped to the same calltemp register. + MOZ_ASSERT(elements == ToRegister(apply->getArgc())); // Invariants guarded in the caller: // - the array is not too long // - the array length equals its initialized length // The array length is our argc for the purposes of allocating space. - Address length(ToRegister(apply->getElements()), - ObjectElements::offsetOfLength()); - masm.load32(length, tmpArgc); + masm.load32(Address(elements, ObjectElements::offsetOfLength()), tmpArgc); // Allocate space for the values. emitAllocateSpaceForApply(tmpArgc, scratch); // After this call "elements" has become "argc". size_t elementsOffset = 0; - emitPushArrayAsArguments(tmpArgc, elementsAndArgc, scratch, elementsOffset); + emitPushArrayAsArguments(tmpArgc, elements, scratch, elementsOffset); // Push |this|. masm.pushValue(ToValue(apply, LApplyArrayGeneric::ThisIndex)); } -void CodeGenerator::emitPushArguments(LConstructArgsGeneric* construct, - Register scratch) { - MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget())); - - // Holds the function nargs. Initially the number of args to the caller. +void CodeGenerator::emitPushArguments(LConstructArgsGeneric* construct) { + // Holds the function nargs. Register argcreg = ToRegister(construct->getArgc()); Register copyreg = ToRegister(construct->getTempObject()); + Register scratch = ToRegister(construct->getTempForArgCopy()); uint32_t extraFormals = construct->numExtraFormals(); + // newTarget and scratch are mapped to the same calltemp register. + MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget())); + // Allocate space for the values. // After this call "newTarget" has become "scratch". emitAllocateSpaceForConstructAndPushNewTarget(argcreg, scratch); @@ -6639,29 +6663,31 @@ void CodeGenerator::emitPushArguments(LConstructArgsGeneric* construct, masm.pushValue(ToValue(construct, LConstructArgsGeneric::ThisIndex)); } -void CodeGenerator::emitPushArguments(LConstructArrayGeneric* construct, - Register scratch) { - MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget())); - +void CodeGenerator::emitPushArguments(LConstructArrayGeneric* construct) { + Register elements = ToRegister(construct->getElements()); Register tmpArgc = ToRegister(construct->getTempObject()); - Register elementsAndArgc = ToRegister(construct->getElements()); + Register scratch = ToRegister(construct->getTempForArgCopy()); + + // argc and elements are mapped to the same calltemp register. + MOZ_ASSERT(elements == ToRegister(construct->getArgc())); + + // newTarget and scratch are mapped to the same calltemp register. + MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget())); // Invariants guarded in the caller: // - the array is not too long // - the array length equals its initialized length // The array length is our argc for the purposes of allocating space. - Address length(ToRegister(construct->getElements()), - ObjectElements::offsetOfLength()); - masm.load32(length, tmpArgc); + masm.load32(Address(elements, ObjectElements::offsetOfLength()), tmpArgc); // Allocate space for the values. + // After this call "newTarget" has become "scratch". emitAllocateSpaceForConstructAndPushNewTarget(tmpArgc, scratch); - // After this call "elements" has become "argc" and "newTarget" has become - // "scratch". + // After this call "elements" has become "argc". size_t elementsOffset = 0; - emitPushArrayAsArguments(tmpArgc, elementsAndArgc, scratch, elementsOffset); + emitPushArrayAsArguments(tmpArgc, elements, scratch, elementsOffset); // Push |this|. masm.pushValue(ToValue(construct, LConstructArrayGeneric::ThisIndex)); @@ -6682,43 +6708,24 @@ void CodeGenerator::emitApplyGeneric(T* apply) { // Copy the arguments of the current function. // - // In the case of ApplyArray, ConstructArray, or ApplyArgsObj, also - // compute argc. The argc register and the elements/argsObj register - // are the same; argc must not be referenced before the call to - // emitPushArguments() and elements/argsObj must not be referenced - // after it returns. + // In the case of ApplyArray, ConstructArray, or ApplyArgsObj, also compute + // argc. The argc register and the elements/argsObj register are the same; + // argc must not be referenced before the call to emitPushArguments() and + // elements/argsObj must not be referenced after it returns. // - // In the case of ConstructArray or ConstructArgs, also overwrite newTarget - // with scratch; newTarget must not be referenced after this point. + // In the case of ConstructArray or ConstructArgs, also overwrite newTarget; + // newTarget must not be referenced after this point. // // objreg is dead across this call. - emitPushArguments(apply, scratch); + emitPushArguments(apply); masm.checkStackAlignment(); bool constructing = apply->mir()->isConstructing(); - // If the function is native, only emit the call to InvokeFunction. - if (apply->hasSingleTarget() && - apply->getSingleTarget()->isNativeWithoutJitEntry()) { - emitCallInvokeFunction(apply); - -#ifdef DEBUG - // Native constructors are guaranteed to return an Object value, so we never - // have to replace a primitive result with the previously allocated Object - // from CreateThis. - if (constructing) { - Label notPrimitive; - masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, - ¬Primitive); - masm.assumeUnreachable("native constructors don't return primitives"); - masm.bind(¬Primitive); - } -#endif - - emitRestoreStackPointerFromFP(); - return; - } + // If the function is native, the call is compiled through emitApplyNative. + MOZ_ASSERT_IF(apply->hasSingleTarget(), + !apply->getSingleTarget()->isNativeWithoutJitEntry()); Label end, invoke; @@ -6812,8 +6819,8 @@ void CodeGenerator::emitApplyGeneric(T* apply) { masm.bind(&end); - // If the return value of the constructing function is Primitive, - // replace the return value with the Object from CreateThis. + // If the return value of the constructing function is Primitive, replace the + // return value with the Object from CreateThis. if (constructing) { Label notPrimitive; masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, @@ -6833,17 +6840,200 @@ void CodeGenerator::emitApplyGeneric(T* apply) { emitRestoreStackPointerFromFP(); } -void CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply) { +template +void CodeGenerator::emitCallInvokeNativeFunction(T* apply) { + pushArg(masm.getStackPointer()); // argv. + pushArg(ToRegister(apply->getArgc())); // argc. + pushArg(Imm32(apply->mir()->ignoresReturnValue())); // ignoresReturnValue. + pushArg(Imm32(apply->mir()->isConstructing())); // isConstructing. + + using Fn = + bool (*)(JSContext*, bool, bool, uint32_t, Value*, MutableHandleValue); + callVM(apply); +} + +template +void CodeGenerator::emitPushNativeArguments(T* apply) { + Register argc = ToRegister(apply->getArgc()); + Register tmpArgc = ToRegister(apply->getTempObject()); + Register scratch = ToRegister(apply->getTempForArgCopy()); + uint32_t extraFormals = apply->numExtraFormals(); + + // Push arguments. + Label noCopy; + masm.branchTestPtr(Assembler::Zero, argc, argc, &noCopy); + { + // Use scratch register to calculate stack space (no padding needed). + masm.movePtr(argc, scratch); + + // Reserve space for copying the arguments. + NativeObject::elementsSizeMustNotOverflow(); + masm.lshiftPtr(Imm32(ValueShift), scratch); + masm.subFromStackPtr(scratch); + + // Compute the source and destination offsets into the stack. + Register argvSrcBase = FramePointer; + size_t argvSrcOffset = + JitFrameLayout::offsetOfActualArgs() + extraFormals * sizeof(JS::Value); + size_t argvDstOffset = 0; + + Register argvIndex = tmpArgc; + masm.move32(argc, argvIndex); + + // Copy arguments. + emitCopyValuesForApply(argvSrcBase, argvIndex, scratch, argvSrcOffset, + argvDstOffset); + } + masm.bind(&noCopy); +} + +template +void CodeGenerator::emitPushArrayAsNativeArguments(T* apply) { + Register argc = ToRegister(apply->getArgc()); + Register elements = ToRegister(apply->getElements()); + Register tmpArgc = ToRegister(apply->getTempObject()); + Register scratch = ToRegister(apply->getTempForArgCopy()); + + // NB: argc and elements are mapped to the same register. + MOZ_ASSERT(argc == elements); + + // Invariants guarded in the caller: + // - the array is not too long + // - the array length equals its initialized length + + // The array length is our argc. + masm.load32(Address(elements, ObjectElements::offsetOfLength()), tmpArgc); + + // Skip the copy of arguments if there are none. + Label noCopy; + masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy); + { + // |tmpArgc| is off-by-one, so adjust the offset accordingly. + BaseObjectElementIndex srcPtr(elements, tmpArgc, + -int32_t(sizeof(JS::Value))); + + Label loop; + masm.bind(&loop); + masm.pushValue(srcPtr, scratch); + masm.decBranchPtr(Assembler::NonZero, tmpArgc, Imm32(1), &loop); + } + masm.bind(&noCopy); + + // Set argc in preparation for emitCallInvokeNativeFunction. + masm.load32(Address(elements, ObjectElements::offsetOfLength()), argc); +} + +void CodeGenerator::emitPushArguments(LApplyArgsNative* apply) { + emitPushNativeArguments(apply); +} + +void CodeGenerator::emitPushArguments(LApplyArrayNative* apply) { + emitPushArrayAsNativeArguments(apply); +} + +void CodeGenerator::emitPushArguments(LConstructArgsNative* construct) { + emitPushNativeArguments(construct); +} + +void CodeGenerator::emitPushArguments(LConstructArrayNative* construct) { + emitPushArrayAsNativeArguments(construct); +} + +void CodeGenerator::emitPushArguments(LApplyArgsObjNative* apply) { + Register argc = ToRegister(apply->getArgc()); + Register argsObj = ToRegister(apply->getArgsObj()); + Register tmpArgc = ToRegister(apply->getTempObject()); + Register scratch = ToRegister(apply->getTempForArgCopy()); + + // NB: argc and argsObj are mapped to the same register. + MOZ_ASSERT(argc == argsObj); + + // Load argc into tmpArgc. + masm.loadArgumentsObjectLength(argsObj, tmpArgc); + + // Push arguments. + Label noCopy, epilogue; + masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy); + { + // Use scratch register to calculate stack space (no padding needed). + masm.movePtr(tmpArgc, scratch); + + // Reserve space for copying the arguments. + NativeObject::elementsSizeMustNotOverflow(); + masm.lshiftPtr(Imm32(ValueShift), scratch); + masm.subFromStackPtr(scratch); + + // Load arguments data. + Register argvSrcBase = argsObj; + masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), + argvSrcBase); + size_t argvSrcOffset = ArgumentsData::offsetOfArgs(); + size_t argvDstOffset = 0; + + // Stash away |tmpArgc| and adjust argvDstOffset accordingly. + masm.push(tmpArgc); + argvDstOffset += sizeof(void*); + + // Copy the values. + emitCopyValuesForApply(argvSrcBase, tmpArgc, scratch, argvSrcOffset, + argvDstOffset); + + // Set argc in preparation for emitCallInvokeNativeFunction. + masm.pop(argc); + masm.jump(&epilogue); + } + masm.bind(&noCopy); + { + // Set argc in preparation for emitCallInvokeNativeFunction. + masm.movePtr(ImmWord(0), argc); + } + masm.bind(&epilogue); +} + +template +void CodeGenerator::emitApplyNative(T* apply) { + MOZ_ASSERT(apply->mir()->getSingleTarget()->isNativeWithoutJitEntry()); + + constexpr bool isConstructing = T::isConstructing(); + MOZ_ASSERT(isConstructing == apply->mir()->isConstructing(), + "isConstructing condition must be consistent"); + + // Push newTarget. + if constexpr (isConstructing) { + masm.pushValue(JSVAL_TYPE_OBJECT, ToRegister(apply->getNewTarget())); + } + + // Push arguments. + emitPushArguments(apply); + + // Push |this|. + if constexpr (isConstructing) { + masm.pushValue(MagicValue(JS_IS_CONSTRUCTING)); + } else { + masm.pushValue(ToValue(apply, T::ThisIndex)); + } + + // Push callee. + masm.pushValue(JSVAL_TYPE_OBJECT, ToRegister(apply->getFunction())); + + // Call the native function. + emitCallInvokeNativeFunction(apply); + + // Pop arguments and continue. + emitRestoreStackPointerFromFP(); +} + +template +void CodeGenerator::emitApplyArgsGuard(T* apply) { LSnapshot* snapshot = apply->snapshot(); Register argcreg = ToRegister(apply->getArgc()); // Ensure that we have a reasonable number of arguments. bailoutCmp32(Assembler::Above, argcreg, Imm32(JIT_ARGS_LENGTH_MAX), snapshot); - - emitApplyGeneric(apply); } -void CodeGenerator::visitApplyArgsObj(LApplyArgsObj* apply) { +template +void CodeGenerator::emitApplyArgsObjGuard(T* apply) { Register argsObj = ToRegister(apply->getArgsObj()); Register temp = ToRegister(apply->getTempObject()); @@ -6851,16 +7041,15 @@ void CodeGenerator::visitApplyArgsObj(LApplyArgsObj* apply) { masm.loadArgumentsObjectLength(argsObj, temp, &bail); masm.branch32(Assembler::Above, temp, Imm32(JIT_ARGS_LENGTH_MAX), &bail); bailoutFrom(&bail, apply->snapshot()); - - emitApplyGeneric(apply); } -void CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply) { +template +void CodeGenerator::emitApplyArrayGuard(T* apply) { LSnapshot* snapshot = apply->snapshot(); + Register elements = ToRegister(apply->getElements()); Register tmp = ToRegister(apply->getTempObject()); - Address length(ToRegister(apply->getElements()), - ObjectElements::offsetOfLength()); + Address length(elements, ObjectElements::offsetOfLength()); masm.load32(length, tmp); // Ensure that we have a reasonable number of arguments. @@ -6868,43 +7057,60 @@ void CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply) { // Ensure that the array does not contain an uninitialized tail. - Address initializedLength(ToRegister(apply->getElements()), + Address initializedLength(elements, ObjectElements::offsetOfInitializedLength()); masm.sub32(initializedLength, tmp); bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot); +} +void CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply) { + emitApplyArgsGuard(apply); emitApplyGeneric(apply); } -void CodeGenerator::visitConstructArgsGeneric(LConstructArgsGeneric* lir) { - LSnapshot* snapshot = lir->snapshot(); - Register argcreg = ToRegister(lir->getArgc()); +void CodeGenerator::visitApplyArgsObj(LApplyArgsObj* apply) { + emitApplyArgsObjGuard(apply); + emitApplyGeneric(apply); +} - // Ensure that we have a reasonable number of arguments. - bailoutCmp32(Assembler::Above, argcreg, Imm32(JIT_ARGS_LENGTH_MAX), snapshot); +void CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply) { + emitApplyArrayGuard(apply); + emitApplyGeneric(apply); +} +void CodeGenerator::visitConstructArgsGeneric(LConstructArgsGeneric* lir) { + emitApplyArgsGuard(lir); emitApplyGeneric(lir); } void CodeGenerator::visitConstructArrayGeneric(LConstructArrayGeneric* lir) { - LSnapshot* snapshot = lir->snapshot(); - Register tmp = ToRegister(lir->getTempObject()); + emitApplyArrayGuard(lir); + emitApplyGeneric(lir); +} - Address length(ToRegister(lir->getElements()), - ObjectElements::offsetOfLength()); - masm.load32(length, tmp); +void CodeGenerator::visitApplyArgsNative(LApplyArgsNative* lir) { + emitApplyArgsGuard(lir); + emitApplyNative(lir); +} - // Ensure that we have a reasonable number of arguments. - bailoutCmp32(Assembler::Above, tmp, Imm32(JIT_ARGS_LENGTH_MAX), snapshot); +void CodeGenerator::visitApplyArgsObjNative(LApplyArgsObjNative* lir) { + emitApplyArgsObjGuard(lir); + emitApplyNative(lir); +} - // Ensure that the array does not contain an uninitialized tail. +void CodeGenerator::visitApplyArrayNative(LApplyArrayNative* lir) { + emitApplyArrayGuard(lir); + emitApplyNative(lir); +} - Address initializedLength(ToRegister(lir->getElements()), - ObjectElements::offsetOfInitializedLength()); - masm.sub32(initializedLength, tmp); - bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot); +void CodeGenerator::visitConstructArgsNative(LConstructArgsNative* lir) { + emitApplyArgsGuard(lir); + emitApplyNative(lir); +} - emitApplyGeneric(lir); +void CodeGenerator::visitConstructArrayNative(LConstructArrayNative* lir) { + emitApplyArrayGuard(lir); + emitApplyNative(lir); } void CodeGenerator::visitBail(LBail* lir) { bailout(lir->snapshot()); } @@ -15460,15 +15666,37 @@ void CodeGenerator::validateAndRegisterFuseDependencies(JSContext* cx, if (!hasSeenObjectEmulateUndefinedFuse.intact()) { JitSpew(JitSpew_Codegen, - "tossing compilation; fuse dependency no longer valid\n"); + "tossing compilation; hasSeenObjectEmulateUndefinedFuse fuse " + "dependency no longer valid\n"); *isValid = false; return; } if (!hasSeenObjectEmulateUndefinedFuse.addFuseDependency(cx, script)) { - JitSpew( - JitSpew_Codegen, - "tossing compilation; failed to register script dependency\n"); + JitSpew(JitSpew_Codegen, + "tossing compilation; failed to register " + "hasSeenObjectEmulateUndefinedFuse script dependency\n"); + *isValid = false; + return; + } + break; + } + + case FuseDependencyKind::OptimizeGetIteratorFuse: { + auto& optimizeGetIteratorFuse = + cx->realm()->realmFuses.optimizeGetIteratorFuse; + if (!optimizeGetIteratorFuse.intact()) { + JitSpew(JitSpew_Codegen, + "tossing compilation; optimizeGetIteratorFuse fuse " + "dependency no longer valid\n"); + *isValid = false; + return; + } + + if (!optimizeGetIteratorFuse.addFuseDependency(cx, script)) { + JitSpew(JitSpew_Codegen, + "tossing compilation; failed to register " + "optimizeGetIteratorFuse script dependency\n"); *isValid = false; return; } @@ -15837,15 +16065,16 @@ void CodeGenerator::visitMegamorphicSetElement(LMegamorphicSetElement* lir) { void CodeGenerator::visitLoadScriptedProxyHandler( LLoadScriptedProxyHandler* ins) { - const Register obj = ToRegister(ins->getOperand(0)); - ValueOperand output = ToOutValue(ins); + Register obj = ToRegister(ins->getOperand(0)); + Register output = ToRegister(ins->output()); - masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), - output.scratchReg()); - masm.loadValue( - Address(output.scratchReg(), js::detail::ProxyReservedSlots::offsetOfSlot( - ScriptedProxyHandler::HANDLER_EXTRA)), - output); + masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), output); + + Label bail; + Address handlerAddr(output, js::detail::ProxyReservedSlots::offsetOfSlot( + ScriptedProxyHandler::HANDLER_EXTRA)); + masm.fallibleUnboxObject(handlerAddr, output, &bail); + bailoutFrom(&bail, ins->snapshot()); } #ifdef JS_PUNBOX64 @@ -19861,9 +20090,17 @@ void CodeGenerator::visitLoadWrapperTarget(LLoadWrapperTarget* lir) { Register output = ToRegister(lir->output()); masm.loadPtr(Address(object, ProxyObject::offsetOfReservedSlots()), output); - masm.unboxObject( - Address(output, js::detail::ProxyReservedSlots::offsetOfPrivateSlot()), - output); + + // Bail for revoked proxies. + Label bail; + Address targetAddr(output, + js::detail::ProxyReservedSlots::offsetOfPrivateSlot()); + if (lir->mir()->fallible()) { + masm.fallibleUnboxObject(targetAddr, output, &bail); + bailoutFrom(&bail, lir->snapshot()); + } else { + masm.unboxObject(targetAddr, output); + } } void CodeGenerator::visitGuardHasGetterSetter(LGuardHasGetterSetter* lir) { @@ -20642,9 +20879,20 @@ void CodeGenerator::visitWasmAnyRefFromJSString(LWasmAnyRefFromJSString* lir) { } void CodeGenerator::visitWasmNewI31Ref(LWasmNewI31Ref* lir) { - Register value = ToRegister(lir->value()); - Register output = ToRegister(lir->output()); - masm.truncate32ToWasmI31Ref(value, output); + if (lir->value()->isConstant()) { + // i31ref are often created with constants. If that's the case we will + // do the operation statically here. This is similar to what is done + // in masm.truncate32ToWasmI31Ref. + Register output = ToRegister(lir->output()); + uint32_t value = + static_cast(lir->value()->toConstant()->toInt32()); + uintptr_t ptr = wasm::AnyRef::fromUint32Truncate(value).rawValue(); + masm.movePtr(ImmWord(ptr), output); + } else { + Register value = ToRegister(lir->value()); + Register output = ToRegister(lir->output()); + masm.truncate32ToWasmI31Ref(value, output); + } } void CodeGenerator::visitWasmI31RefGet(LWasmI31RefGet* lir) { -- cgit v1.2.3