/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- * vim: set ts=8 sts=2 et sw=2 tw=80: * * Copyright 2017 Mozilla Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "wasm/WasmBuiltins.h" #include "mozilla/Atomics.h" #include "fdlibm.h" #include "jslibmath.h" #include "jsmath.h" #include "gc/Allocator.h" #include "jit/AtomicOperations.h" #include "jit/InlinableNatives.h" #include "jit/MacroAssembler.h" #include "jit/Simulator.h" #include "js/experimental/JitInfo.h" // JSJitInfo #include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_* #include "js/friend/StackLimits.h" // js::CheckRecursionLimit #include "threading/Mutex.h" #include "util/Memory.h" #include "util/Poison.h" #include "vm/BigIntType.h" #include "wasm/WasmInstance.h" #include "wasm/WasmStubs.h" #include "wasm/WasmTypes.h" #include "debugger/DebugAPI-inl.h" #include "vm/Stack-inl.h" using namespace js; using namespace jit; using namespace wasm; using mozilla::HashGeneric; using mozilla::IsNaN; using mozilla::MakeEnumeratedRange; static const unsigned BUILTIN_THUNK_LIFO_SIZE = 64 * 1024; // ============================================================================ // WebAssembly builtin C++ functions called from wasm code to implement internal // wasm operations: type descriptions. // Some abbreviations, for the sake of conciseness. #define _F64 MIRType::Double #define _F32 MIRType::Float32 #define _I32 MIRType::Int32 #define _I64 MIRType::Int64 #define _PTR MIRType::Pointer #define _RoN MIRType::RefOrNull #define _VOID MIRType::None #define _END MIRType::None #define _Infallible FailureMode::Infallible #define _FailOnNegI32 FailureMode::FailOnNegI32 #define _FailOnNullPtr FailureMode::FailOnNullPtr #define _FailOnInvalidRef FailureMode::FailOnInvalidRef namespace js { namespace wasm { const SymbolicAddressSignature SASigSinD = { SymbolicAddress::SinD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigCosD = { SymbolicAddress::CosD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigTanD = { SymbolicAddress::TanD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigASinD = { SymbolicAddress::ASinD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigACosD = { SymbolicAddress::ACosD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigATanD = { SymbolicAddress::ATanD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigCeilD = { SymbolicAddress::CeilD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigCeilF = { SymbolicAddress::CeilF, _F32, _Infallible, 1, {_F32, _END}}; const SymbolicAddressSignature SASigFloorD = { SymbolicAddress::FloorD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigFloorF = { SymbolicAddress::FloorF, _F32, _Infallible, 1, {_F32, _END}}; const SymbolicAddressSignature SASigTruncD = { SymbolicAddress::TruncD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigTruncF = { SymbolicAddress::TruncF, _F32, _Infallible, 1, {_F32, _END}}; const SymbolicAddressSignature SASigNearbyIntD = { SymbolicAddress::NearbyIntD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigNearbyIntF = { SymbolicAddress::NearbyIntF, _F32, _Infallible, 1, {_F32, _END}}; const SymbolicAddressSignature SASigExpD = { SymbolicAddress::ExpD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigLogD = { SymbolicAddress::LogD, _F64, _Infallible, 1, {_F64, _END}}; const SymbolicAddressSignature SASigPowD = { SymbolicAddress::PowD, _F64, _Infallible, 2, {_F64, _F64, _END}}; const SymbolicAddressSignature SASigATan2D = { SymbolicAddress::ATan2D, _F64, _Infallible, 2, {_F64, _F64, _END}}; const SymbolicAddressSignature SASigMemoryGrow = { SymbolicAddress::MemoryGrow, _I32, _Infallible, 2, {_PTR, _I32, _END}}; const SymbolicAddressSignature SASigMemorySize = { SymbolicAddress::MemorySize, _I32, _Infallible, 1, {_PTR, _END}}; const SymbolicAddressSignature SASigWaitI32 = {SymbolicAddress::WaitI32, _I32, _FailOnNegI32, 4, {_PTR, _I32, _I32, _I64, _END}}; const SymbolicAddressSignature SASigWaitI64 = {SymbolicAddress::WaitI64, _I32, _FailOnNegI32, 4, {_PTR, _I32, _I64, _I64, _END}}; const SymbolicAddressSignature SASigWake = { SymbolicAddress::Wake, _I32, _FailOnNegI32, 3, {_PTR, _I32, _I32, _END}}; const SymbolicAddressSignature SASigMemCopy = { SymbolicAddress::MemCopy, _VOID, _FailOnNegI32, 5, {_PTR, _I32, _I32, _I32, _PTR, _END}}; const SymbolicAddressSignature SASigMemCopyShared = { SymbolicAddress::MemCopyShared, _VOID, _FailOnNegI32, 5, {_PTR, _I32, _I32, _I32, _PTR, _END}}; const SymbolicAddressSignature SASigDataDrop = { SymbolicAddress::DataDrop, _VOID, _FailOnNegI32, 2, {_PTR, _I32, _END}}; const SymbolicAddressSignature SASigMemFill = { SymbolicAddress::MemFill, _VOID, _FailOnNegI32, 5, {_PTR, _I32, _I32, _I32, _PTR, _END}}; const SymbolicAddressSignature SASigMemFillShared = { SymbolicAddress::MemFillShared, _VOID, _FailOnNegI32, 5, {_PTR, _I32, _I32, _I32, _PTR, _END}}; const SymbolicAddressSignature SASigMemInit = { SymbolicAddress::MemInit, _VOID, _FailOnNegI32, 5, {_PTR, _I32, _I32, _I32, _I32, _END}}; const SymbolicAddressSignature SASigTableCopy = { SymbolicAddress::TableCopy, _VOID, _FailOnNegI32, 6, {_PTR, _I32, _I32, _I32, _I32, _I32, _END}}; const SymbolicAddressSignature SASigElemDrop = { SymbolicAddress::ElemDrop, _VOID, _FailOnNegI32, 2, {_PTR, _I32, _END}}; const SymbolicAddressSignature SASigTableFill = { SymbolicAddress::TableFill, _VOID, _FailOnNegI32, 5, {_PTR, _I32, _RoN, _I32, _I32, _END}}; const SymbolicAddressSignature SASigTableGet = {SymbolicAddress::TableGet, _RoN, _FailOnInvalidRef, 3, {_PTR, _I32, _I32, _END}}; const SymbolicAddressSignature SASigTableGrow = { SymbolicAddress::TableGrow, _I32, _Infallible, 4, {_PTR, _RoN, _I32, _I32, _END}}; const SymbolicAddressSignature SASigTableInit = { SymbolicAddress::TableInit, _VOID, _FailOnNegI32, 6, {_PTR, _I32, _I32, _I32, _I32, _I32, _END}}; const SymbolicAddressSignature SASigTableSet = {SymbolicAddress::TableSet, _VOID, _FailOnNegI32, 4, {_PTR, _I32, _RoN, _I32, _END}}; const SymbolicAddressSignature SASigTableSize = { SymbolicAddress::TableSize, _I32, _Infallible, 2, {_PTR, _I32, _END}}; const SymbolicAddressSignature SASigRefFunc = { SymbolicAddress::RefFunc, _RoN, _FailOnInvalidRef, 2, {_PTR, _I32, _END}}; const SymbolicAddressSignature SASigPreBarrierFiltering = { SymbolicAddress::PreBarrierFiltering, _VOID, _Infallible, 2, {_PTR, _PTR, _END}}; const SymbolicAddressSignature SASigPostBarrier = { SymbolicAddress::PostBarrier, _VOID, _Infallible, 2, {_PTR, _PTR, _END}}; const SymbolicAddressSignature SASigPostBarrierFiltering = { SymbolicAddress::PostBarrierFiltering, _VOID, _Infallible, 2, {_PTR, _PTR, _END}}; const SymbolicAddressSignature SASigStructNew = { SymbolicAddress::StructNew, _RoN, _FailOnNullPtr, 2, {_PTR, _RoN, _END}}; const SymbolicAddressSignature SASigStructNarrow = { SymbolicAddress::StructNarrow, _RoN, _Infallible, 3, {_PTR, _RoN, _RoN, _END}}; } // namespace wasm } // namespace js #undef _F64 #undef _F32 #undef _I32 #undef _I64 #undef _PTR #undef _RoN #undef _VOID #undef _END #undef _Infallible #undef _FailOnNegI32 #undef _FailOnNullPtr #ifdef DEBUG ABIArgType ToABIType(FailureMode mode) { switch (mode) { case FailureMode::FailOnNegI32: return ArgType_Int32; case FailureMode::FailOnNullPtr: case FailureMode::FailOnInvalidRef: return ArgType_General; default: MOZ_CRASH("unexpected failure mode"); } } ABIArgType ToABIType(MIRType type) { switch (type) { case MIRType::None: case MIRType::Int32: return ArgType_Int32; case MIRType::Int64: return ArgType_Int64; case MIRType::Pointer: case MIRType::RefOrNull: return ArgType_General; case MIRType::Float32: return ArgType_Float32; case MIRType::Double: return ArgType_Float64; default: MOZ_CRASH("unexpected type"); } } ABIFunctionType ToABIType(const SymbolicAddressSignature& sig) { MOZ_ASSERT_IF(sig.failureMode != FailureMode::Infallible, ToABIType(sig.failureMode) == ToABIType(sig.retType)); int abiType = ToABIType(sig.retType) << RetType_Shift; for (int i = 0; i < sig.numArgs; i++) { abiType |= (ToABIType(sig.argTypes[i]) << (ArgType_Shift * (i + 1))); } return ABIFunctionType(abiType); } #endif // ============================================================================ // WebAssembly builtin C++ functions called from wasm code to implement internal // wasm operations: implementations. #if defined(JS_CODEGEN_ARM) extern "C" { extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int); extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int); } #endif // This utility function can only be called for builtins that are called // directly from wasm code. static JitActivation* CallingActivation() { Activation* act = TlsContext.get()->activation(); MOZ_ASSERT(act->asJit()->hasWasmExitFP()); return act->asJit(); } static bool WasmHandleDebugTrap() { JitActivation* activation = CallingActivation(); JSContext* cx = activation->cx(); Frame* fp = activation->wasmExitFP(); Instance* instance = GetNearestEffectiveTls(fp)->instance; const Code& code = instance->code(); MOZ_ASSERT(code.metadata().debugEnabled); // The debug trap stub is the innermost frame. It's return address is the // actual trap site. const CallSite* site = code.lookupCallSite(fp->returnAddress()); MOZ_ASSERT(site); // Advance to the actual trapping frame. fp = fp->wasmCaller(); DebugFrame* debugFrame = DebugFrame::from(fp); if (site->kind() == CallSite::EnterFrame) { if (!instance->debug().enterFrameTrapsEnabled()) { return true; } debugFrame->setIsDebuggee(); debugFrame->observe(cx); if (!DebugAPI::onEnterFrame(cx, debugFrame)) { if (cx->isPropagatingForcedReturn()) { cx->clearPropagatingForcedReturn(); // Ignoring forced return because changing code execution order is // not yet implemented in the wasm baseline. // TODO properly handle forced return and resume wasm execution. JS_ReportErrorASCII(cx, "Unexpected resumption value from onEnterFrame"); } return false; } return true; } if (site->kind() == CallSite::LeaveFrame) { if (!debugFrame->updateReturnJSValue(cx)) { return false; } bool ok = DebugAPI::onLeaveFrame(cx, debugFrame, nullptr, true); debugFrame->leave(cx); return ok; } DebugState& debug = instance->debug(); MOZ_ASSERT(debug.hasBreakpointTrapAtOffset(site->lineOrBytecode())); if (debug.stepModeEnabled(debugFrame->funcIndex())) { if (!DebugAPI::onSingleStep(cx)) { if (cx->isPropagatingForcedReturn()) { cx->clearPropagatingForcedReturn(); // TODO properly handle forced return. JS_ReportErrorASCII(cx, "Unexpected resumption value from onSingleStep"); } return false; } } if (debug.hasBreakpointSite(site->lineOrBytecode())) { if (!DebugAPI::onTrap(cx)) { if (cx->isPropagatingForcedReturn()) { cx->clearPropagatingForcedReturn(); // TODO properly handle forced return. JS_ReportErrorASCII( cx, "Unexpected resumption value from breakpoint handler"); } return false; } } return true; } // Unwind the entire activation in response to a thrown exception. This function // is responsible for notifying the debugger of each unwound frame. The return // value is the new stack address which the calling stub will set to the sp // register before executing a return instruction. void* wasm::HandleThrow(JSContext* cx, WasmFrameIter& iter) { // WasmFrameIter iterates down wasm frames in the activation starting at // JitActivation::wasmExitFP(). Calling WasmFrameIter::startUnwinding pops // JitActivation::wasmExitFP() once each time WasmFrameIter is incremented, // ultimately leaving exit FP null when the WasmFrameIter is done(). This // is necessary to prevent a DebugFrame from being observed again after we // just called onLeaveFrame (which would lead to the frame being re-added // to the map of live frames, right as it becomes trash). MOZ_ASSERT(CallingActivation() == iter.activation()); MOZ_ASSERT(!iter.done()); iter.setUnwind(WasmFrameIter::Unwind::True); // Live wasm code on the stack is kept alive (in TraceJitActivation) by // marking the instance of every wasm::Frame found by WasmFrameIter. // However, as explained above, we're popping frames while iterating which // means that a GC during this loop could collect the code of frames whose // code is still on the stack. This is actually mostly fine: as soon as we // return to the throw stub, the entire stack will be popped as a whole, // returning to the C++ caller. However, we must keep the throw stub alive // itself which is owned by the innermost instance. RootedWasmInstanceObject keepAlive(cx, iter.instance()->object()); for (; !iter.done(); ++iter) { // Wasm code can enter same-compartment realms, so reset cx->realm to // this frame's realm. cx->setRealmForJitExceptionHandler(iter.instance()->realm()); if (!iter.debugEnabled()) { continue; } DebugFrame* frame = iter.debugFrame(); frame->clearReturnJSValue(); // Assume ResumeMode::Terminate if no exception is pending -- // no onExceptionUnwind handlers must be fired. if (cx->isExceptionPending()) { if (!DebugAPI::onExceptionUnwind(cx, frame)) { if (cx->isPropagatingForcedReturn()) { cx->clearPropagatingForcedReturn(); // Unexpected trap return -- raising error since throw recovery // is not yet implemented in the wasm baseline. // TODO properly handle forced return and resume wasm execution. JS_ReportErrorASCII( cx, "Unexpected resumption value from onExceptionUnwind"); } } } bool ok = DebugAPI::onLeaveFrame(cx, frame, nullptr, false); if (ok) { // Unexpected success from the handler onLeaveFrame -- raising error // since throw recovery is not yet implemented in the wasm baseline. // TODO properly handle success and resume wasm execution. JS_ReportErrorASCII(cx, "Unexpected success from onLeaveFrame"); } frame->leave(cx); } MOZ_ASSERT(!cx->activation()->asJit()->isWasmTrapping(), "unwinding clears the trapping state"); return iter.unwoundAddressOfReturnAddress(); } static void* WasmHandleThrow() { JitActivation* activation = CallingActivation(); JSContext* cx = activation->cx(); WasmFrameIter iter(activation); return HandleThrow(cx, iter); } // Unconditionally returns nullptr per calling convention of HandleTrap(). static void* ReportError(JSContext* cx, unsigned errorNumber) { JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, errorNumber); return nullptr; }; // Has the same return-value convention as HandleTrap(). static void* CheckInterrupt(JSContext* cx, JitActivation* activation) { ResetInterruptState(cx); if (!CheckForInterrupt(cx)) { return nullptr; } void* resumePC = activation->wasmTrapData().resumePC; activation->finishWasmTrap(); return resumePC; } // The calling convention between this function and its caller in the stub // generated by GenerateTrapExit() is: // - return nullptr if the stub should jump to the throw stub to unwind // the activation; // - return the (non-null) resumePC that should be jumped if execution should // resume after the trap. static void* WasmHandleTrap() { JitActivation* activation = CallingActivation(); JSContext* cx = activation->cx(); switch (activation->wasmTrapData().trap) { case Trap::Unreachable: return ReportError(cx, JSMSG_WASM_UNREACHABLE); case Trap::IntegerOverflow: return ReportError(cx, JSMSG_WASM_INTEGER_OVERFLOW); case Trap::InvalidConversionToInteger: return ReportError(cx, JSMSG_WASM_INVALID_CONVERSION); case Trap::IntegerDivideByZero: return ReportError(cx, JSMSG_WASM_INT_DIVIDE_BY_ZERO); case Trap::IndirectCallToNull: return ReportError(cx, JSMSG_WASM_IND_CALL_TO_NULL); case Trap::IndirectCallBadSig: return ReportError(cx, JSMSG_WASM_IND_CALL_BAD_SIG); case Trap::NullPointerDereference: return ReportError(cx, JSMSG_WASM_DEREF_NULL); case Trap::OutOfBounds: return ReportError(cx, JSMSG_WASM_OUT_OF_BOUNDS); case Trap::UnalignedAccess: return ReportError(cx, JSMSG_WASM_UNALIGNED_ACCESS); case Trap::CheckInterrupt: return CheckInterrupt(cx, activation); case Trap::StackOverflow: // TlsData::setInterrupt() causes a fake stack overflow. Since // TlsData::setInterrupt() is called racily, it's possible for a real // stack overflow to trap, followed by a racy call to setInterrupt(). // Thus, we must check for a real stack overflow first before we // CheckInterrupt() and possibly resume execution. if (!CheckRecursionLimit(cx)) { return nullptr; } if (activation->wasmExitTls()->isInterrupted()) { return CheckInterrupt(cx, activation); } return ReportError(cx, JSMSG_OVER_RECURSED); case Trap::ThrowReported: // Error was already reported under another name. return nullptr; case Trap::Limit: break; } MOZ_CRASH("unexpected trap"); } static void WasmReportV128JSCall() { JSContext* cx = TlsContext.get(); JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_VAL_TYPE); } static int32_t CoerceInPlace_ToInt32(Value* rawVal) { JSContext* cx = TlsContext.get(); int32_t i32; RootedValue val(cx, *rawVal); if (!ToInt32(cx, val, &i32)) { *rawVal = PoisonedObjectValue(0x42); return false; } *rawVal = Int32Value(i32); return true; } static int32_t CoerceInPlace_ToBigInt(Value* rawVal) { JSContext* cx = TlsContext.get(); RootedValue val(cx, *rawVal); BigInt* bi = ToBigInt(cx, val); if (!bi) { *rawVal = PoisonedObjectValue(0x43); return false; } *rawVal = BigIntValue(bi); return true; } static int32_t CoerceInPlace_ToNumber(Value* rawVal) { JSContext* cx = TlsContext.get(); double dbl; RootedValue val(cx, *rawVal); if (!ToNumber(cx, val, &dbl)) { *rawVal = PoisonedObjectValue(0x42); return false; } *rawVal = DoubleValue(dbl); return true; } static void* BoxValue_Anyref(Value* rawVal) { JSContext* cx = TlsContext.get(); RootedValue val(cx, *rawVal); RootedAnyRef result(cx, AnyRef::null()); if (!BoxAnyRef(cx, val, &result)) { return nullptr; } return result.get().forCompiledCode(); } static int32_t CoerceInPlace_JitEntry(int funcExportIndex, TlsData* tlsData, Value* argv) { JSContext* cx = CallingActivation()->cx(); const Code& code = tlsData->instance->code(); const FuncExport& fe = code.metadata(code.stableTier()).funcExports[funcExportIndex]; for (size_t i = 0; i < fe.funcType().args().length(); i++) { HandleValue arg = HandleValue::fromMarkedLocation(&argv[i]); switch (fe.funcType().args()[i].kind()) { case ValType::I32: { int32_t i32; if (!ToInt32(cx, arg, &i32)) { return false; } argv[i] = Int32Value(i32); break; } case ValType::I64: { // In this case we store a BigInt value as there is no value type // corresponding directly to an I64. The conversion to I64 happens // in the JIT entry stub. BigInt* bigint = ToBigInt(cx, arg); if (!bigint) { return false; } argv[i] = BigIntValue(bigint); break; } case ValType::F32: case ValType::F64: { double dbl; if (!ToNumber(cx, arg, &dbl)) { return false; } // No need to convert double-to-float for f32, it's done inline // in the wasm stub later. argv[i] = DoubleValue(dbl); break; } case ValType::Ref: { switch (fe.funcType().args()[i].refTypeKind()) { case RefType::Extern: // Leave Object and Null alone, we will unbox inline. All we need // to do is convert other values to an Object representation. if (!arg.isObjectOrNull()) { RootedAnyRef result(cx, AnyRef::null()); if (!BoxAnyRef(cx, arg, &result)) { return false; } argv[i].setObject(*result.get().asJSObject()); } break; case RefType::Func: case RefType::Eq: case RefType::TypeIndex: // Guarded against by temporarilyUnsupportedReftypeForEntry() MOZ_CRASH("unexpected input argument in CoerceInPlace_JitEntry"); } break; } case ValType::V128: { // Guarded against by hasV128ArgOrRet() MOZ_CRASH("unexpected input argument in CoerceInPlace_JitEntry"); } default: { MOZ_CRASH("unexpected input argument in CoerceInPlace_JitEntry"); } } } return true; } // Allocate a BigInt without GC, corresponds to the similar VMFunction. static BigInt* AllocateBigIntTenuredNoGC() { JSContext* cx = TlsContext.get(); return js::AllocateBigInt(cx, gc::TenuredHeap); } static int64_t DivI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, uint32_t y_lo) { int64_t x = ((uint64_t)x_hi << 32) + x_lo; int64_t y = ((uint64_t)y_hi << 32) + y_lo; MOZ_ASSERT(x != INT64_MIN || y != -1); MOZ_ASSERT(y != 0); return x / y; } static int64_t UDivI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, uint32_t y_lo) { uint64_t x = ((uint64_t)x_hi << 32) + x_lo; uint64_t y = ((uint64_t)y_hi << 32) + y_lo; MOZ_ASSERT(y != 0); return x / y; } static int64_t ModI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, uint32_t y_lo) { int64_t x = ((uint64_t)x_hi << 32) + x_lo; int64_t y = ((uint64_t)y_hi << 32) + y_lo; MOZ_ASSERT(x != INT64_MIN || y != -1); MOZ_ASSERT(y != 0); return x % y; } static int64_t UModI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, uint32_t y_lo) { uint64_t x = ((uint64_t)x_hi << 32) + x_lo; uint64_t y = ((uint64_t)y_hi << 32) + y_lo; MOZ_ASSERT(y != 0); return x % y; } static int64_t TruncateDoubleToInt64(double input) { // Note: INT64_MAX is not representable in double. It is actually // INT64_MAX + 1. Therefore also sending the failure value. if (input >= double(INT64_MAX) || input < double(INT64_MIN) || IsNaN(input)) { return 0x8000000000000000; } return int64_t(input); } static uint64_t TruncateDoubleToUint64(double input) { // Note: UINT64_MAX is not representable in double. It is actually // UINT64_MAX + 1. Therefore also sending the failure value. if (input >= double(UINT64_MAX) || input <= -1.0 || IsNaN(input)) { return 0x8000000000000000; } return uint64_t(input); } static int64_t SaturatingTruncateDoubleToInt64(double input) { // Handle in-range values (except INT64_MIN). if (fabs(input) < -double(INT64_MIN)) { return int64_t(input); } // Handle NaN. if (IsNaN(input)) { return 0; } // Handle positive overflow. if (input > 0) { return INT64_MAX; } // Handle negative overflow. return INT64_MIN; } static uint64_t SaturatingTruncateDoubleToUint64(double input) { // Handle positive overflow. if (input >= -double(INT64_MIN) * 2.0) { return UINT64_MAX; } // Handle in-range values. if (input > -1.0) { return uint64_t(input); } // Handle NaN and negative overflow. return 0; } static double Int64ToDouble(int32_t x_hi, uint32_t x_lo) { int64_t x = int64_t((uint64_t(x_hi) << 32)) + int64_t(x_lo); return double(x); } static float Int64ToFloat32(int32_t x_hi, uint32_t x_lo) { int64_t x = int64_t((uint64_t(x_hi) << 32)) + int64_t(x_lo); return float(x); } static double Uint64ToDouble(int32_t x_hi, uint32_t x_lo) { uint64_t x = (uint64_t(x_hi) << 32) + uint64_t(x_lo); return double(x); } static float Uint64ToFloat32(int32_t x_hi, uint32_t x_lo) { uint64_t x = (uint64_t(x_hi) << 32) + uint64_t(x_lo); return float(x); } template static inline void* FuncCast(F* funcPtr, ABIFunctionType abiType) { void* pf = JS_FUNC_TO_DATA_PTR(void*, funcPtr); #ifdef JS_SIMULATOR pf = Simulator::RedirectNativeFunction(pf, abiType); #endif return pf; } #ifdef WASM_CODEGEN_DEBUG void wasm::PrintI32(int32_t val) { fprintf(stderr, "i32(%d) ", val); } void wasm::PrintPtr(uint8_t* val) { fprintf(stderr, "ptr(%p) ", val); } void wasm::PrintF32(float val) { fprintf(stderr, "f32(%f) ", val); } void wasm::PrintF64(double val) { fprintf(stderr, "f64(%lf) ", val); } void wasm::PrintText(const char* out) { fprintf(stderr, "%s", out); } #endif void* wasm::AddressOf(SymbolicAddress imm, ABIFunctionType* abiType) { switch (imm) { case SymbolicAddress::HandleDebugTrap: *abiType = Args_General0; return FuncCast(WasmHandleDebugTrap, *abiType); case SymbolicAddress::HandleThrow: *abiType = Args_General0; return FuncCast(WasmHandleThrow, *abiType); case SymbolicAddress::HandleTrap: *abiType = Args_General0; return FuncCast(WasmHandleTrap, *abiType); case SymbolicAddress::ReportV128JSCall: *abiType = Args_General0; return FuncCast(WasmReportV128JSCall, *abiType); case SymbolicAddress::CallImport_General: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_General}); return FuncCast(Instance::callImport_general, *abiType); case SymbolicAddress::CoerceInPlace_ToInt32: *abiType = Args_General1; return FuncCast(CoerceInPlace_ToInt32, *abiType); case SymbolicAddress::CoerceInPlace_ToBigInt: *abiType = Args_General1; return FuncCast(CoerceInPlace_ToBigInt, *abiType); case SymbolicAddress::CoerceInPlace_ToNumber: *abiType = Args_General1; return FuncCast(CoerceInPlace_ToNumber, *abiType); case SymbolicAddress::CoerceInPlace_JitEntry: *abiType = Args_General3; return FuncCast(CoerceInPlace_JitEntry, *abiType); case SymbolicAddress::ToInt32: *abiType = Args_Int_Double; return FuncCast(JS::ToInt32, *abiType); case SymbolicAddress::BoxValue_Anyref: *abiType = Args_General1; return FuncCast(BoxValue_Anyref, *abiType); case SymbolicAddress::AllocateBigInt: *abiType = Args_General0; return FuncCast(AllocateBigIntTenuredNoGC, *abiType); case SymbolicAddress::DivI64: *abiType = Args_General4; return FuncCast(DivI64, *abiType); case SymbolicAddress::UDivI64: *abiType = Args_General4; return FuncCast(UDivI64, *abiType); case SymbolicAddress::ModI64: *abiType = Args_General4; return FuncCast(ModI64, *abiType); case SymbolicAddress::UModI64: *abiType = Args_General4; return FuncCast(UModI64, *abiType); case SymbolicAddress::TruncateDoubleToUint64: *abiType = Args_Int64_Double; return FuncCast(TruncateDoubleToUint64, *abiType); case SymbolicAddress::TruncateDoubleToInt64: *abiType = Args_Int64_Double; return FuncCast(TruncateDoubleToInt64, *abiType); case SymbolicAddress::SaturatingTruncateDoubleToUint64: *abiType = Args_Int64_Double; return FuncCast(SaturatingTruncateDoubleToUint64, *abiType); case SymbolicAddress::SaturatingTruncateDoubleToInt64: *abiType = Args_Int64_Double; return FuncCast(SaturatingTruncateDoubleToInt64, *abiType); case SymbolicAddress::Uint64ToDouble: *abiType = Args_Double_IntInt; return FuncCast(Uint64ToDouble, *abiType); case SymbolicAddress::Uint64ToFloat32: *abiType = Args_Float32_IntInt; return FuncCast(Uint64ToFloat32, *abiType); case SymbolicAddress::Int64ToDouble: *abiType = Args_Double_IntInt; return FuncCast(Int64ToDouble, *abiType); case SymbolicAddress::Int64ToFloat32: *abiType = Args_Float32_IntInt; return FuncCast(Int64ToFloat32, *abiType); #if defined(JS_CODEGEN_ARM) case SymbolicAddress::aeabi_idivmod: *abiType = Args_General2; return FuncCast(__aeabi_idivmod, *abiType); case SymbolicAddress::aeabi_uidivmod: *abiType = Args_General2; return FuncCast(__aeabi_uidivmod, *abiType); #endif case SymbolicAddress::ModD: *abiType = Args_Double_DoubleDouble; return FuncCast(NumberMod, *abiType); case SymbolicAddress::SinD: *abiType = Args_Double_Double; return FuncCast(sin, *abiType); case SymbolicAddress::CosD: *abiType = Args_Double_Double; return FuncCast(cos, *abiType); case SymbolicAddress::TanD: *abiType = Args_Double_Double; return FuncCast(tan, *abiType); case SymbolicAddress::ASinD: *abiType = Args_Double_Double; return FuncCast(fdlibm::asin, *abiType); case SymbolicAddress::ACosD: *abiType = Args_Double_Double; return FuncCast(fdlibm::acos, *abiType); case SymbolicAddress::ATanD: *abiType = Args_Double_Double; return FuncCast(fdlibm::atan, *abiType); case SymbolicAddress::CeilD: *abiType = Args_Double_Double; return FuncCast(fdlibm::ceil, *abiType); case SymbolicAddress::CeilF: *abiType = Args_Float32_Float32; return FuncCast(fdlibm::ceilf, *abiType); case SymbolicAddress::FloorD: *abiType = Args_Double_Double; return FuncCast(fdlibm::floor, *abiType); case SymbolicAddress::FloorF: *abiType = Args_Float32_Float32; return FuncCast(fdlibm::floorf, *abiType); case SymbolicAddress::TruncD: *abiType = Args_Double_Double; return FuncCast(fdlibm::trunc, *abiType); case SymbolicAddress::TruncF: *abiType = Args_Float32_Float32; return FuncCast(fdlibm::truncf, *abiType); case SymbolicAddress::NearbyIntD: *abiType = Args_Double_Double; return FuncCast(fdlibm::nearbyint, *abiType); case SymbolicAddress::NearbyIntF: *abiType = Args_Float32_Float32; return FuncCast(fdlibm::nearbyintf, *abiType); case SymbolicAddress::ExpD: *abiType = Args_Double_Double; return FuncCast(fdlibm::exp, *abiType); case SymbolicAddress::LogD: *abiType = Args_Double_Double; return FuncCast(fdlibm::log, *abiType); case SymbolicAddress::PowD: *abiType = Args_Double_DoubleDouble; return FuncCast(ecmaPow, *abiType); case SymbolicAddress::ATan2D: *abiType = Args_Double_DoubleDouble; return FuncCast(ecmaAtan2, *abiType); case SymbolicAddress::MemoryGrow: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigMemoryGrow)); return FuncCast(Instance::memoryGrow_i32, *abiType); case SymbolicAddress::MemorySize: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigMemorySize)); return FuncCast(Instance::memorySize_i32, *abiType); case SymbolicAddress::WaitI32: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int64}); MOZ_ASSERT(*abiType == ToABIType(SASigWaitI32)); return FuncCast(Instance::wait_i32, *abiType); case SymbolicAddress::WaitI64: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int64, ArgType_Int64}); MOZ_ASSERT(*abiType == ToABIType(SASigWaitI64)); return FuncCast(Instance::wait_i64, *abiType); case SymbolicAddress::Wake: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigWake)); return FuncCast(Instance::wake, *abiType); case SymbolicAddress::MemCopy: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigMemCopy)); return FuncCast(Instance::memCopy, *abiType); case SymbolicAddress::MemCopyShared: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigMemCopyShared)); return FuncCast(Instance::memCopyShared, *abiType); case SymbolicAddress::DataDrop: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigDataDrop)); return FuncCast(Instance::dataDrop, *abiType); case SymbolicAddress::MemFill: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigMemFill)); return FuncCast(Instance::memFill, *abiType); case SymbolicAddress::MemFillShared: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigMemFillShared)); return FuncCast(Instance::memFillShared, *abiType); case SymbolicAddress::MemInit: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigMemInit)); return FuncCast(Instance::memInit, *abiType); case SymbolicAddress::TableCopy: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableCopy)); return FuncCast(Instance::tableCopy, *abiType); case SymbolicAddress::ElemDrop: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigElemDrop)); return FuncCast(Instance::elemDrop, *abiType); case SymbolicAddress::TableFill: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_General, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableFill)); return FuncCast(Instance::tableFill, *abiType); case SymbolicAddress::TableInit: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableInit)); return FuncCast(Instance::tableInit, *abiType); case SymbolicAddress::TableGet: *abiType = MakeABIFunctionType( ArgType_General, {ArgType_General, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableGet)); return FuncCast(Instance::tableGet, *abiType); case SymbolicAddress::TableGrow: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_General, ArgType_Int32, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableGrow)); return FuncCast(Instance::tableGrow, *abiType); case SymbolicAddress::TableSet: *abiType = MakeABIFunctionType( ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_General, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableSet)); return FuncCast(Instance::tableSet, *abiType); case SymbolicAddress::TableSize: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigTableSize)); return FuncCast(Instance::tableSize, *abiType); case SymbolicAddress::RefFunc: *abiType = MakeABIFunctionType(ArgType_General, {ArgType_General, ArgType_Int32}); MOZ_ASSERT(*abiType == ToABIType(SASigRefFunc)); return FuncCast(Instance::refFunc, *abiType); case SymbolicAddress::PostBarrier: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigPostBarrier)); return FuncCast(Instance::postBarrier, *abiType); case SymbolicAddress::PreBarrierFiltering: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigPreBarrierFiltering)); return FuncCast(Instance::preBarrierFiltering, *abiType); case SymbolicAddress::PostBarrierFiltering: *abiType = MakeABIFunctionType(ArgType_Int32, {ArgType_General, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigPostBarrierFiltering)); return FuncCast(Instance::postBarrierFiltering, *abiType); case SymbolicAddress::StructNew: *abiType = MakeABIFunctionType(ArgType_General, {ArgType_General, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigStructNew)); return FuncCast(Instance::structNew, *abiType); case SymbolicAddress::StructNarrow: *abiType = MakeABIFunctionType( ArgType_General, {ArgType_General, ArgType_General, ArgType_General}); MOZ_ASSERT(*abiType == ToABIType(SASigStructNarrow)); return FuncCast(Instance::structNarrow, *abiType); #if defined(JS_CODEGEN_MIPS32) case SymbolicAddress::js_jit_gAtomic64Lock: return &js::jit::gAtomic64Lock; #endif #ifdef WASM_CODEGEN_DEBUG case SymbolicAddress::PrintI32: *abiType = Args_General1; return FuncCast(PrintI32, *abiType); case SymbolicAddress::PrintPtr: *abiType = Args_General1; return FuncCast(PrintPtr, *abiType); case SymbolicAddress::PrintF32: *abiType = Args_Int_Float32; return FuncCast(PrintF32, *abiType); case SymbolicAddress::PrintF64: *abiType = Args_Int_Double; return FuncCast(PrintF64, *abiType); case SymbolicAddress::PrintText: *abiType = Args_General1; return FuncCast(PrintText, *abiType); #endif case SymbolicAddress::Limit: break; } MOZ_CRASH("Bad SymbolicAddress"); } bool wasm::NeedsBuiltinThunk(SymbolicAddress sym) { // Some functions don't want to a thunk, because they already have one or // they don't have frame info. switch (sym) { case SymbolicAddress::HandleDebugTrap: // GenerateDebugTrapStub case SymbolicAddress::HandleThrow: // GenerateThrowStub case SymbolicAddress::HandleTrap: // GenerateTrapExit case SymbolicAddress::CallImport_General: // GenerateImportInterpExit case SymbolicAddress::CoerceInPlace_ToInt32: // GenerateImportJitExit case SymbolicAddress::CoerceInPlace_ToNumber: case SymbolicAddress::CoerceInPlace_ToBigInt: case SymbolicAddress::BoxValue_Anyref: #if defined(JS_CODEGEN_MIPS32) case SymbolicAddress::js_jit_gAtomic64Lock: #endif #ifdef WASM_CODEGEN_DEBUG case SymbolicAddress::PrintI32: case SymbolicAddress::PrintPtr: case SymbolicAddress::PrintF32: case SymbolicAddress::PrintF64: case SymbolicAddress::PrintText: // Used only in stubs #endif return false; case SymbolicAddress::ToInt32: case SymbolicAddress::DivI64: case SymbolicAddress::UDivI64: case SymbolicAddress::ModI64: case SymbolicAddress::UModI64: case SymbolicAddress::TruncateDoubleToUint64: case SymbolicAddress::TruncateDoubleToInt64: case SymbolicAddress::SaturatingTruncateDoubleToUint64: case SymbolicAddress::SaturatingTruncateDoubleToInt64: case SymbolicAddress::Uint64ToDouble: case SymbolicAddress::Uint64ToFloat32: case SymbolicAddress::Int64ToDouble: case SymbolicAddress::Int64ToFloat32: #if defined(JS_CODEGEN_ARM) case SymbolicAddress::aeabi_idivmod: case SymbolicAddress::aeabi_uidivmod: #endif case SymbolicAddress::AllocateBigInt: case SymbolicAddress::ModD: case SymbolicAddress::SinD: case SymbolicAddress::CosD: case SymbolicAddress::TanD: case SymbolicAddress::ASinD: case SymbolicAddress::ACosD: case SymbolicAddress::ATanD: case SymbolicAddress::CeilD: case SymbolicAddress::CeilF: case SymbolicAddress::FloorD: case SymbolicAddress::FloorF: case SymbolicAddress::TruncD: case SymbolicAddress::TruncF: case SymbolicAddress::NearbyIntD: case SymbolicAddress::NearbyIntF: case SymbolicAddress::ExpD: case SymbolicAddress::LogD: case SymbolicAddress::PowD: case SymbolicAddress::ATan2D: case SymbolicAddress::MemoryGrow: case SymbolicAddress::MemorySize: case SymbolicAddress::WaitI32: case SymbolicAddress::WaitI64: case SymbolicAddress::Wake: case SymbolicAddress::CoerceInPlace_JitEntry: case SymbolicAddress::ReportV128JSCall: case SymbolicAddress::MemCopy: case SymbolicAddress::MemCopyShared: case SymbolicAddress::DataDrop: case SymbolicAddress::MemFill: case SymbolicAddress::MemFillShared: case SymbolicAddress::MemInit: case SymbolicAddress::TableCopy: case SymbolicAddress::ElemDrop: case SymbolicAddress::TableFill: case SymbolicAddress::TableGet: case SymbolicAddress::TableGrow: case SymbolicAddress::TableInit: case SymbolicAddress::TableSet: case SymbolicAddress::TableSize: case SymbolicAddress::RefFunc: case SymbolicAddress::PreBarrierFiltering: case SymbolicAddress::PostBarrier: case SymbolicAddress::PostBarrierFiltering: case SymbolicAddress::StructNew: case SymbolicAddress::StructNarrow: return true; case SymbolicAddress::Limit: break; } MOZ_CRASH("unexpected symbolic address"); } // ============================================================================ // JS builtins that can be imported by wasm modules and called efficiently // through thunks. These thunks conform to the internal wasm ABI and thus can be // patched in for import calls. Calling a JS builtin through a thunk is much // faster than calling out through the generic import call trampoline which will // end up in the slowest C++ Instance::callImport path. // // Each JS builtin can have several overloads. These must all be enumerated in // PopulateTypedNatives() so they can be included in the process-wide thunk set. #define FOR_EACH_UNARY_NATIVE(_) \ _(math_sin, MathSin) \ _(math_tan, MathTan) \ _(math_cos, MathCos) \ _(math_exp, MathExp) \ _(math_log, MathLog) \ _(math_asin, MathASin) \ _(math_atan, MathATan) \ _(math_acos, MathACos) \ _(math_log10, MathLog10) \ _(math_log2, MathLog2) \ _(math_log1p, MathLog1P) \ _(math_expm1, MathExpM1) \ _(math_sinh, MathSinH) \ _(math_tanh, MathTanH) \ _(math_cosh, MathCosH) \ _(math_asinh, MathASinH) \ _(math_atanh, MathATanH) \ _(math_acosh, MathACosH) \ _(math_sign, MathSign) \ _(math_trunc, MathTrunc) \ _(math_cbrt, MathCbrt) #define FOR_EACH_BINARY_NATIVE(_) \ _(ecmaAtan2, MathATan2) \ _(ecmaHypot, MathHypot) \ _(ecmaPow, MathPow) #define DEFINE_UNARY_FLOAT_WRAPPER(func, _) \ static float func##_impl_f32(float x) { \ return float(func##_impl(double(x))); \ } #define DEFINE_BINARY_FLOAT_WRAPPER(func, _) \ static float func##_f32(float x, float y) { \ return float(func(double(x), double(y))); \ } FOR_EACH_UNARY_NATIVE(DEFINE_UNARY_FLOAT_WRAPPER) FOR_EACH_BINARY_NATIVE(DEFINE_BINARY_FLOAT_WRAPPER) #undef DEFINE_UNARY_FLOAT_WRAPPER #undef DEFINE_BINARY_FLOAT_WRAPPER struct TypedNative { InlinableNative native; ABIFunctionType abiType; TypedNative(InlinableNative native, ABIFunctionType abiType) : native(native), abiType(abiType) {} using Lookup = TypedNative; static HashNumber hash(const Lookup& l) { return HashGeneric(uint32_t(l.native), uint32_t(l.abiType)); } static bool match(const TypedNative& lhs, const Lookup& rhs) { return lhs.native == rhs.native && lhs.abiType == rhs.abiType; } }; using TypedNativeToFuncPtrMap = HashMap; static bool PopulateTypedNatives(TypedNativeToFuncPtrMap* typedNatives) { #define ADD_OVERLOAD(funcName, native, abiType) \ if (!typedNatives->putNew(TypedNative(InlinableNative::native, abiType), \ FuncCast(funcName, abiType))) \ return false; #define ADD_UNARY_OVERLOADS(funcName, native) \ ADD_OVERLOAD(funcName##_impl, native, Args_Double_Double) \ ADD_OVERLOAD(funcName##_impl_f32, native, Args_Float32_Float32) #define ADD_BINARY_OVERLOADS(funcName, native) \ ADD_OVERLOAD(funcName, native, Args_Double_DoubleDouble) \ ADD_OVERLOAD(funcName##_f32, native, Args_Float32_Float32Float32) FOR_EACH_UNARY_NATIVE(ADD_UNARY_OVERLOADS) FOR_EACH_BINARY_NATIVE(ADD_BINARY_OVERLOADS) #undef ADD_UNARY_OVERLOADS #undef ADD_BINARY_OVERLOADS return true; } #undef FOR_EACH_UNARY_NATIVE #undef FOR_EACH_BINARY_NATIVE // ============================================================================ // Process-wide builtin thunk set // // Thunks are inserted between wasm calls and the C++ callee and achieve two // things: // - bridging the few differences between the internal wasm ABI and the // external native ABI (viz. float returns on x86 and soft-fp ARM) // - executing an exit prologue/epilogue which in turn allows any profiling // iterator to see the full stack up to the wasm operation that called out // // Thunks are created for two kinds of C++ callees, enumerated above: // - SymbolicAddress: for statically compiled calls in the wasm module // - Imported JS builtins: optimized calls to imports // // All thunks are created up front, lazily, when the first wasm module is // compiled in the process. Thunks are kept alive until the JS engine shuts down // in the process. No thunks are created at runtime after initialization. This // simple scheme allows several simplifications: // - no reference counting to keep thunks alive // - no problems toggling W^X permissions which, because of multiple executing // threads, would require each thunk allocation to be on its own page // The cost for creating all thunks at once is relatively low since all thunks // fit within the smallest executable quanta (64k). using TypedNativeToCodeRangeMap = HashMap; using SymbolicAddressToCodeRangeArray = EnumeratedArray; struct BuiltinThunks { uint8_t* codeBase; size_t codeSize; CodeRangeVector codeRanges; TypedNativeToCodeRangeMap typedNativeToCodeRange; SymbolicAddressToCodeRangeArray symbolicAddressToCodeRange; uint32_t provisionalJitEntryOffset; BuiltinThunks() : codeBase(nullptr), codeSize(0) {} ~BuiltinThunks() { if (codeBase) { DeallocateExecutableMemory(codeBase, codeSize); } } }; Mutex initBuiltinThunks(mutexid::WasmInitBuiltinThunks); Atomic builtinThunks; bool wasm::EnsureBuiltinThunksInitialized() { LockGuard guard(initBuiltinThunks); if (builtinThunks) { return true; } auto thunks = MakeUnique(); if (!thunks) { return false; } LifoAlloc lifo(BUILTIN_THUNK_LIFO_SIZE); TempAllocator tempAlloc(&lifo); WasmMacroAssembler masm(tempAlloc); for (auto sym : MakeEnumeratedRange(SymbolicAddress::Limit)) { if (!NeedsBuiltinThunk(sym)) { thunks->symbolicAddressToCodeRange[sym] = UINT32_MAX; continue; } uint32_t codeRangeIndex = thunks->codeRanges.length(); thunks->symbolicAddressToCodeRange[sym] = codeRangeIndex; ABIFunctionType abiType; void* funcPtr = AddressOf(sym, &abiType); ExitReason exitReason(sym); CallableOffsets offsets; if (!GenerateBuiltinThunk(masm, abiType, exitReason, funcPtr, &offsets)) { return false; } if (!thunks->codeRanges.emplaceBack(CodeRange::BuiltinThunk, offsets)) { return false; } } TypedNativeToFuncPtrMap typedNatives; if (!PopulateTypedNatives(&typedNatives)) { return false; } for (TypedNativeToFuncPtrMap::Range r = typedNatives.all(); !r.empty(); r.popFront()) { TypedNative typedNative = r.front().key(); uint32_t codeRangeIndex = thunks->codeRanges.length(); if (!thunks->typedNativeToCodeRange.putNew(typedNative, codeRangeIndex)) { return false; } ABIFunctionType abiType = typedNative.abiType; void* funcPtr = r.front().value(); ExitReason exitReason = ExitReason::Fixed::BuiltinNative; CallableOffsets offsets; if (!GenerateBuiltinThunk(masm, abiType, exitReason, funcPtr, &offsets)) { return false; } if (!thunks->codeRanges.emplaceBack(CodeRange::BuiltinThunk, offsets)) { return false; } } // Provisional JitEntry stub: This is a shared stub that can be installed in // the jit-entry jump table. It uses the JIT ABI and when invoked will // retrieve (via TlsContext()) and invoke the context-appropriate // invoke-from-interpreter jit stub, thus serving as the initial, unoptimized // jit-entry stub for any exported wasm function that has a jit-entry. #ifdef DEBUG // We need to allow this machine code to bake in a C++ code pointer, so we // disable the wasm restrictions while generating this stub. JitContext jitContext(&tempAlloc); bool oldFlag = jitContext.setIsCompilingWasm(false); #endif Offsets provisionalJitEntryOffsets; if (!GenerateProvisionalJitEntryStub(masm, &provisionalJitEntryOffsets)) { return false; } thunks->provisionalJitEntryOffset = provisionalJitEntryOffsets.begin; #ifdef DEBUG jitContext.setIsCompilingWasm(oldFlag); #endif masm.finish(); if (masm.oom()) { return false; } size_t allocSize = AlignBytes(masm.bytesNeeded(), ExecutableCodePageSize); thunks->codeSize = allocSize; thunks->codeBase = (uint8_t*)AllocateExecutableMemory( allocSize, ProtectionSetting::Writable, MemCheckKind::MakeUndefined); if (!thunks->codeBase) { return false; } masm.executableCopy(thunks->codeBase); memset(thunks->codeBase + masm.bytesNeeded(), 0, allocSize - masm.bytesNeeded()); masm.processCodeLabels(thunks->codeBase); PatchDebugSymbolicAccesses(thunks->codeBase, masm); MOZ_ASSERT(masm.callSites().empty()); MOZ_ASSERT(masm.callSiteTargets().empty()); MOZ_ASSERT(masm.trapSites().empty()); if (!ExecutableAllocator::makeExecutableAndFlushICache( FlushICacheSpec::LocalThreadOnly, thunks->codeBase, thunks->codeSize)) { return false; } builtinThunks = thunks.release(); return true; } void wasm::ReleaseBuiltinThunks() { if (builtinThunks) { const BuiltinThunks* ptr = builtinThunks; js_delete(const_cast(ptr)); builtinThunks = nullptr; } } void* wasm::SymbolicAddressTarget(SymbolicAddress sym) { MOZ_ASSERT(builtinThunks); ABIFunctionType abiType; void* funcPtr = AddressOf(sym, &abiType); if (!NeedsBuiltinThunk(sym)) { return funcPtr; } const BuiltinThunks& thunks = *builtinThunks; uint32_t codeRangeIndex = thunks.symbolicAddressToCodeRange[sym]; return thunks.codeBase + thunks.codeRanges[codeRangeIndex].begin(); } void* wasm::ProvisionalJitEntryStub() { MOZ_ASSERT(builtinThunks); const BuiltinThunks& thunks = *builtinThunks; return thunks.codeBase + thunks.provisionalJitEntryOffset; } static Maybe ToBuiltinABIFunctionType( const FuncType& funcType) { const ValTypeVector& args = funcType.args(); const ValTypeVector& results = funcType.results(); if (results.length() != 1) { return Nothing(); } uint32_t abiType; switch (results[0].kind()) { case ValType::F32: abiType = ArgType_Float32 << RetType_Shift; break; case ValType::F64: abiType = ArgType_Float64 << RetType_Shift; break; default: return Nothing(); } if ((args.length() + 1) > (sizeof(uint32_t) * 8 / ArgType_Shift)) { return Nothing(); } for (size_t i = 0; i < args.length(); i++) { switch (args[i].kind()) { case ValType::F32: abiType |= (ArgType_Float32 << (ArgType_Shift * (i + 1))); break; case ValType::F64: abiType |= (ArgType_Float64 << (ArgType_Shift * (i + 1))); break; default: return Nothing(); } } return Some(ABIFunctionType(abiType)); } void* wasm::MaybeGetBuiltinThunk(JSFunction* f, const FuncType& funcType) { MOZ_ASSERT(builtinThunks); if (!f->isNative() || !f->hasJitInfo() || f->jitInfo()->type() != JSJitInfo::InlinableNative) { return nullptr; } Maybe abiType = ToBuiltinABIFunctionType(funcType); if (!abiType) { return nullptr; } TypedNative typedNative(f->jitInfo()->inlinableNative, *abiType); const BuiltinThunks& thunks = *builtinThunks; auto p = thunks.typedNativeToCodeRange.readonlyThreadsafeLookup(typedNative); if (!p) { return nullptr; } return thunks.codeBase + thunks.codeRanges[p->value()].begin(); } bool wasm::LookupBuiltinThunk(void* pc, const CodeRange** codeRange, uint8_t** codeBase) { if (!builtinThunks) { return false; } const BuiltinThunks& thunks = *builtinThunks; if (pc < thunks.codeBase || pc >= thunks.codeBase + thunks.codeSize) { return false; } *codeBase = thunks.codeBase; CodeRange::OffsetInCode target((uint8_t*)pc - thunks.codeBase); *codeRange = LookupInSorted(thunks.codeRanges, target); return !!*codeRange; }