From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- js/src/wasm/AsmJS.cpp | 7383 +++++++++++++++++++++++ js/src/wasm/AsmJS.h | 116 + js/src/wasm/GenerateIntrinsics.py | 86 + js/src/wasm/WasmBCClass-inl.h | 58 + js/src/wasm/WasmBCClass.h | 1745 ++++++ js/src/wasm/WasmBCCodegen-inl.h | 524 ++ js/src/wasm/WasmBCDefs.h | 201 + js/src/wasm/WasmBCFrame.cpp | 544 ++ js/src/wasm/WasmBCFrame.h | 1397 +++++ js/src/wasm/WasmBCMemory.cpp | 2799 +++++++++ js/src/wasm/WasmBCRegDefs-inl.h | 180 + js/src/wasm/WasmBCRegDefs.h | 852 +++ js/src/wasm/WasmBCRegMgmt-inl.h | 486 ++ js/src/wasm/WasmBCStk.h | 345 ++ js/src/wasm/WasmBCStkMgmt-inl.h | 1320 ++++ js/src/wasm/WasmBaselineCompile.cpp | 11009 ++++++++++++++++++++++++++++++++++ js/src/wasm/WasmBaselineCompile.h | 103 + js/src/wasm/WasmBinary.cpp | 335 ++ js/src/wasm/WasmBinary.h | 890 +++ js/src/wasm/WasmBuiltins.cpp | 1980 ++++++ js/src/wasm/WasmBuiltins.h | 324 + js/src/wasm/WasmCode.cpp | 1253 ++++ js/src/wasm/WasmCode.h | 874 +++ js/src/wasm/WasmCodegenConstants.h | 76 + js/src/wasm/WasmCodegenTypes.cpp | 240 + js/src/wasm/WasmCodegenTypes.h | 765 +++ js/src/wasm/WasmCompile.cpp | 919 +++ js/src/wasm/WasmCompile.h | 99 + js/src/wasm/WasmCompileArgs.h | 238 + js/src/wasm/WasmConstants.h | 1144 ++++ js/src/wasm/WasmContext.h | 41 + js/src/wasm/WasmDebug.cpp | 528 ++ js/src/wasm/WasmDebug.h | 187 + js/src/wasm/WasmDebugFrame.cpp | 176 + js/src/wasm/WasmDebugFrame.h | 217 + js/src/wasm/WasmException.h | 31 + js/src/wasm/WasmExprType.h | 330 + js/src/wasm/WasmFrame.h | 409 ++ js/src/wasm/WasmFrameIter.cpp | 1764 ++++++ js/src/wasm/WasmFrameIter.h | 277 + js/src/wasm/WasmGC.cpp | 314 + js/src/wasm/WasmGC.h | 495 ++ js/src/wasm/WasmGcObject.cpp | 872 +++ js/src/wasm/WasmGcObject.h | 396 ++ js/src/wasm/WasmGenerator.cpp | 1279 ++++ js/src/wasm/WasmGenerator.h | 264 + js/src/wasm/WasmInitExpr.cpp | 664 ++ js/src/wasm/WasmInitExpr.h | 101 + js/src/wasm/WasmInstance-inl.h | 32 + js/src/wasm/WasmInstance.cpp | 2759 +++++++++ js/src/wasm/WasmInstance.h | 512 ++ js/src/wasm/WasmInstanceData.h | 137 + js/src/wasm/WasmIntrinsic.cpp | 239 + js/src/wasm/WasmIntrinsic.h | 60 + js/src/wasm/WasmIntrinsic.yaml | 201 + js/src/wasm/WasmIonCompile.cpp | 8691 +++++++++++++++++++++++++++ js/src/wasm/WasmIonCompile.h | 41 + js/src/wasm/WasmJS.cpp | 5524 +++++++++++++++++ js/src/wasm/WasmJS.h | 580 ++ js/src/wasm/WasmLog.cpp | 81 + js/src/wasm/WasmLog.h | 51 + js/src/wasm/WasmMemory.cpp | 385 ++ js/src/wasm/WasmMemory.h | 226 + js/src/wasm/WasmModule.cpp | 1134 ++++ js/src/wasm/WasmModule.h | 225 + js/src/wasm/WasmModuleTypes.cpp | 171 + js/src/wasm/WasmModuleTypes.h | 632 ++ js/src/wasm/WasmOpIter.cpp | 863 +++ js/src/wasm/WasmOpIter.h | 4239 +++++++++++++ js/src/wasm/WasmProcess.cpp | 438 ++ js/src/wasm/WasmProcess.h | 76 + js/src/wasm/WasmRealm.cpp | 150 + js/src/wasm/WasmRealm.h | 79 + js/src/wasm/WasmSerialize.cpp | 1230 ++++ js/src/wasm/WasmSerialize.h | 296 + js/src/wasm/WasmShareable.h | 80 + js/src/wasm/WasmSignalHandlers.cpp | 1084 ++++ js/src/wasm/WasmSignalHandlers.h | 63 + js/src/wasm/WasmStubs.cpp | 3117 ++++++++++ js/src/wasm/WasmStubs.h | 370 ++ js/src/wasm/WasmTable.cpp | 473 ++ js/src/wasm/WasmTable.h | 139 + js/src/wasm/WasmTypeDecls.h | 99 + js/src/wasm/WasmTypeDef.cpp | 550 ++ js/src/wasm/WasmTypeDef.h | 1462 +++++ js/src/wasm/WasmUtility.h | 65 + js/src/wasm/WasmValType.cpp | 398 ++ js/src/wasm/WasmValType.h | 890 +++ js/src/wasm/WasmValidate.cpp | 3223 ++++++++++ js/src/wasm/WasmValidate.h | 308 + js/src/wasm/WasmValue.cpp | 927 +++ js/src/wasm/WasmValue.h | 652 ++ js/src/wasm/moz.build | 66 + 93 files changed, 89648 insertions(+) create mode 100644 js/src/wasm/AsmJS.cpp create mode 100644 js/src/wasm/AsmJS.h create mode 100644 js/src/wasm/GenerateIntrinsics.py create mode 100644 js/src/wasm/WasmBCClass-inl.h create mode 100644 js/src/wasm/WasmBCClass.h create mode 100644 js/src/wasm/WasmBCCodegen-inl.h create mode 100644 js/src/wasm/WasmBCDefs.h create mode 100644 js/src/wasm/WasmBCFrame.cpp create mode 100644 js/src/wasm/WasmBCFrame.h create mode 100644 js/src/wasm/WasmBCMemory.cpp create mode 100644 js/src/wasm/WasmBCRegDefs-inl.h create mode 100644 js/src/wasm/WasmBCRegDefs.h create mode 100644 js/src/wasm/WasmBCRegMgmt-inl.h create mode 100644 js/src/wasm/WasmBCStk.h create mode 100644 js/src/wasm/WasmBCStkMgmt-inl.h create mode 100644 js/src/wasm/WasmBaselineCompile.cpp create mode 100644 js/src/wasm/WasmBaselineCompile.h create mode 100644 js/src/wasm/WasmBinary.cpp create mode 100644 js/src/wasm/WasmBinary.h create mode 100644 js/src/wasm/WasmBuiltins.cpp create mode 100644 js/src/wasm/WasmBuiltins.h create mode 100644 js/src/wasm/WasmCode.cpp create mode 100644 js/src/wasm/WasmCode.h create mode 100644 js/src/wasm/WasmCodegenConstants.h create mode 100644 js/src/wasm/WasmCodegenTypes.cpp create mode 100644 js/src/wasm/WasmCodegenTypes.h create mode 100644 js/src/wasm/WasmCompile.cpp create mode 100644 js/src/wasm/WasmCompile.h create mode 100644 js/src/wasm/WasmCompileArgs.h create mode 100644 js/src/wasm/WasmConstants.h create mode 100644 js/src/wasm/WasmContext.h create mode 100644 js/src/wasm/WasmDebug.cpp create mode 100644 js/src/wasm/WasmDebug.h create mode 100644 js/src/wasm/WasmDebugFrame.cpp create mode 100644 js/src/wasm/WasmDebugFrame.h create mode 100644 js/src/wasm/WasmException.h create mode 100644 js/src/wasm/WasmExprType.h create mode 100644 js/src/wasm/WasmFrame.h create mode 100644 js/src/wasm/WasmFrameIter.cpp create mode 100644 js/src/wasm/WasmFrameIter.h create mode 100644 js/src/wasm/WasmGC.cpp create mode 100644 js/src/wasm/WasmGC.h create mode 100644 js/src/wasm/WasmGcObject.cpp create mode 100644 js/src/wasm/WasmGcObject.h create mode 100644 js/src/wasm/WasmGenerator.cpp create mode 100644 js/src/wasm/WasmGenerator.h create mode 100644 js/src/wasm/WasmInitExpr.cpp create mode 100644 js/src/wasm/WasmInitExpr.h create mode 100644 js/src/wasm/WasmInstance-inl.h create mode 100644 js/src/wasm/WasmInstance.cpp create mode 100644 js/src/wasm/WasmInstance.h create mode 100644 js/src/wasm/WasmInstanceData.h create mode 100644 js/src/wasm/WasmIntrinsic.cpp create mode 100644 js/src/wasm/WasmIntrinsic.h create mode 100644 js/src/wasm/WasmIntrinsic.yaml create mode 100644 js/src/wasm/WasmIonCompile.cpp create mode 100644 js/src/wasm/WasmIonCompile.h create mode 100644 js/src/wasm/WasmJS.cpp create mode 100644 js/src/wasm/WasmJS.h create mode 100644 js/src/wasm/WasmLog.cpp create mode 100644 js/src/wasm/WasmLog.h create mode 100644 js/src/wasm/WasmMemory.cpp create mode 100644 js/src/wasm/WasmMemory.h create mode 100644 js/src/wasm/WasmModule.cpp create mode 100644 js/src/wasm/WasmModule.h create mode 100644 js/src/wasm/WasmModuleTypes.cpp create mode 100644 js/src/wasm/WasmModuleTypes.h create mode 100644 js/src/wasm/WasmOpIter.cpp create mode 100644 js/src/wasm/WasmOpIter.h create mode 100644 js/src/wasm/WasmProcess.cpp create mode 100644 js/src/wasm/WasmProcess.h create mode 100644 js/src/wasm/WasmRealm.cpp create mode 100644 js/src/wasm/WasmRealm.h create mode 100644 js/src/wasm/WasmSerialize.cpp create mode 100644 js/src/wasm/WasmSerialize.h create mode 100644 js/src/wasm/WasmShareable.h create mode 100644 js/src/wasm/WasmSignalHandlers.cpp create mode 100644 js/src/wasm/WasmSignalHandlers.h create mode 100644 js/src/wasm/WasmStubs.cpp create mode 100644 js/src/wasm/WasmStubs.h create mode 100644 js/src/wasm/WasmTable.cpp create mode 100644 js/src/wasm/WasmTable.h create mode 100644 js/src/wasm/WasmTypeDecls.h create mode 100644 js/src/wasm/WasmTypeDef.cpp create mode 100644 js/src/wasm/WasmTypeDef.h create mode 100644 js/src/wasm/WasmUtility.h create mode 100644 js/src/wasm/WasmValType.cpp create mode 100644 js/src/wasm/WasmValType.h create mode 100644 js/src/wasm/WasmValidate.cpp create mode 100644 js/src/wasm/WasmValidate.h create mode 100644 js/src/wasm/WasmValue.cpp create mode 100644 js/src/wasm/WasmValue.h create mode 100644 js/src/wasm/moz.build (limited to 'js/src/wasm') diff --git a/js/src/wasm/AsmJS.cpp b/js/src/wasm/AsmJS.cpp new file mode 100644 index 0000000000..c13ceeaac3 --- /dev/null +++ b/js/src/wasm/AsmJS.cpp @@ -0,0 +1,7383 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2014 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/AsmJS.h" + +#include "mozilla/Attributes.h" +#include "mozilla/Compression.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/Maybe.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Sprintf.h" // SprintfLiteral +#include "mozilla/Utf8.h" // mozilla::Utf8Unit +#include "mozilla/Variant.h" + +#include +#include + +#include "jsmath.h" + +#include "frontend/BytecodeCompiler.h" // CompileStandaloneFunction +#include "frontend/FrontendContext.h" // js::FrontendContext +#include "frontend/FunctionSyntaxKind.h" // FunctionSyntaxKind +#include "frontend/ParseNode.h" +#include "frontend/Parser.h" +#include "frontend/ParserAtom.h" // ParserAtomsTable, TaggedParserAtomIndex +#include "frontend/SharedContext.h" // TopLevelFunction +#include "frontend/TaggedParserAtomIndexHasher.h" // TaggedParserAtomIndexHasher +#include "gc/GC.h" +#include "gc/Policy.h" +#include "jit/InlinableNatives.h" +#include "js/BuildId.h" // JS::BuildIdCharVector +#include "js/experimental/JitInfo.h" +#include "js/friend/ErrorMessages.h" // JSMSG_* +#include "js/MemoryMetrics.h" +#include "js/Printf.h" +#include "js/ScalarType.h" // js::Scalar::Type +#include "js/SourceText.h" +#include "js/StableStringChars.h" +#include "js/Wrapper.h" +#include "util/DifferentialTesting.h" +#include "util/StringBuffer.h" +#include "util/Text.h" +#include "vm/ErrorReporting.h" +#include "vm/FunctionFlags.h" // js::FunctionFlags +#include "vm/GeneratorAndAsyncKind.h" // js::GeneratorKind, js::FunctionAsyncKind +#include "vm/Interpreter.h" +#include "vm/SelfHosting.h" +#include "vm/Time.h" +#include "vm/TypedArrayObject.h" +#include "vm/Warnings.h" // js::WarnNumberASCII +#include "wasm/WasmCompile.h" +#include "wasm/WasmGenerator.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmIonCompile.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmValidate.h" + +#include "frontend/SharedContext-inl.h" +#include "vm/ArrayBufferObject-inl.h" +#include "vm/JSObject-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::frontend; +using namespace js::jit; +using namespace js::wasm; + +using JS::AsmJSOption; +using JS::AutoStableStringChars; +using JS::GenericNaN; +using JS::SourceOwnership; +using JS::SourceText; +using mozilla::Abs; +using mozilla::AsVariant; +using mozilla::CeilingLog2; +using mozilla::HashGeneric; +using mozilla::IsNegativeZero; +using mozilla::IsPositiveZero; +using mozilla::IsPowerOfTwo; +using mozilla::Nothing; +using mozilla::PodZero; +using mozilla::PositiveInfinity; +using mozilla::Some; +using mozilla::Utf8Unit; +using mozilla::Compression::LZ4; + +using FunctionVector = JS::GCVector; + +/*****************************************************************************/ + +// A wasm module can either use no memory, a unshared memory (ArrayBuffer) or +// shared memory (SharedArrayBuffer). + +enum class MemoryUsage { None = false, Unshared = 1, Shared = 2 }; + +// The asm.js valid heap lengths are precisely the WASM valid heap lengths for +// ARM greater or equal to MinHeapLength +static const size_t MinHeapLength = PageSize; +// An asm.js heap can in principle be up to INT32_MAX bytes but requirements +// on the format restrict it further to the largest pseudo-ARM-immediate. +// See IsValidAsmJSHeapLength(). +static const uint64_t MaxHeapLength = 0x7f000000; + +static uint64_t RoundUpToNextValidAsmJSHeapLength(uint64_t length) { + if (length <= MinHeapLength) { + return MinHeapLength; + } + + return wasm::RoundUpToNextValidARMImmediate(length); +} + +static uint64_t DivideRoundingUp(uint64_t a, uint64_t b) { + return (a + (b - 1)) / b; +} + +/*****************************************************************************/ +// asm.js module object + +// The asm.js spec recognizes this set of builtin Math functions. +enum AsmJSMathBuiltinFunction { + AsmJSMathBuiltin_sin, + AsmJSMathBuiltin_cos, + AsmJSMathBuiltin_tan, + AsmJSMathBuiltin_asin, + AsmJSMathBuiltin_acos, + AsmJSMathBuiltin_atan, + AsmJSMathBuiltin_ceil, + AsmJSMathBuiltin_floor, + AsmJSMathBuiltin_exp, + AsmJSMathBuiltin_log, + AsmJSMathBuiltin_pow, + AsmJSMathBuiltin_sqrt, + AsmJSMathBuiltin_abs, + AsmJSMathBuiltin_atan2, + AsmJSMathBuiltin_imul, + AsmJSMathBuiltin_fround, + AsmJSMathBuiltin_min, + AsmJSMathBuiltin_max, + AsmJSMathBuiltin_clz32 +}; + +// LitValPOD is a restricted version of LitVal suitable for asm.js that is +// always POD. + +struct LitValPOD { + PackedTypeCode valType_; + union U { + uint32_t u32_; + uint64_t u64_; + float f32_; + double f64_; + } u; + + LitValPOD() = default; + + explicit LitValPOD(uint32_t u32) : valType_(ValType(ValType::I32).packed()) { + u.u32_ = u32; + } + explicit LitValPOD(uint64_t u64) : valType_(ValType(ValType::I64).packed()) { + u.u64_ = u64; + } + + explicit LitValPOD(float f32) : valType_(ValType(ValType::F32).packed()) { + u.f32_ = f32; + } + explicit LitValPOD(double f64) : valType_(ValType(ValType::F64).packed()) { + u.f64_ = f64; + } + + LitVal asLitVal() const { + switch (valType_.typeCode()) { + case TypeCode::I32: + return LitVal(u.u32_); + case TypeCode::I64: + return LitVal(u.u64_); + case TypeCode::F32: + return LitVal(u.f32_); + case TypeCode::F64: + return LitVal(u.f64_); + default: + MOZ_CRASH("Can't happen"); + } + } +}; + +static_assert(std::is_pod_v, + "must be POD to be simply serialized/deserialized"); + +// An AsmJSGlobal represents a JS global variable in the asm.js module function. +class AsmJSGlobal { + public: + enum Which { + Variable, + FFI, + ArrayView, + ArrayViewCtor, + MathBuiltinFunction, + Constant + }; + enum VarInitKind { InitConstant, InitImport }; + enum ConstantKind { GlobalConstant, MathConstant }; + + private: + struct CacheablePod { + Which which_; + union V { + struct { + VarInitKind initKind_; + union U { + PackedTypeCode importValType_; + LitValPOD val_; + } u; + } var; + uint32_t ffiIndex_; + Scalar::Type viewType_; + AsmJSMathBuiltinFunction mathBuiltinFunc_; + struct { + ConstantKind kind_; + double value_; + } constant; + } u; + } pod; + CacheableChars field_; + + friend class ModuleValidatorShared; + template + friend class ModuleValidator; + + public: + AsmJSGlobal() = default; + AsmJSGlobal(Which which, UniqueChars field) { + mozilla::PodZero(&pod); // zero padding for Valgrind + pod.which_ = which; + field_ = std::move(field); + } + const char* field() const { return field_.get(); } + Which which() const { return pod.which_; } + VarInitKind varInitKind() const { + MOZ_ASSERT(pod.which_ == Variable); + return pod.u.var.initKind_; + } + LitValPOD varInitVal() const { + MOZ_ASSERT(pod.which_ == Variable); + MOZ_ASSERT(pod.u.var.initKind_ == InitConstant); + return pod.u.var.u.val_; + } + ValType varInitImportType() const { + MOZ_ASSERT(pod.which_ == Variable); + MOZ_ASSERT(pod.u.var.initKind_ == InitImport); + return ValType(pod.u.var.u.importValType_); + } + uint32_t ffiIndex() const { + MOZ_ASSERT(pod.which_ == FFI); + return pod.u.ffiIndex_; + } + // When a view is created from an imported constructor: + // var I32 = stdlib.Int32Array; + // var i32 = new I32(buffer); + // the second import has nothing to validate and thus has a null field. + Scalar::Type viewType() const { + MOZ_ASSERT(pod.which_ == ArrayView || pod.which_ == ArrayViewCtor); + return pod.u.viewType_; + } + AsmJSMathBuiltinFunction mathBuiltinFunction() const { + MOZ_ASSERT(pod.which_ == MathBuiltinFunction); + return pod.u.mathBuiltinFunc_; + } + ConstantKind constantKind() const { + MOZ_ASSERT(pod.which_ == Constant); + return pod.u.constant.kind_; + } + double constantValue() const { + MOZ_ASSERT(pod.which_ == Constant); + return pod.u.constant.value_; + } +}; + +using AsmJSGlobalVector = Vector; + +// An AsmJSImport is slightly different than an asm.js FFI function: a single +// asm.js FFI function can be called with many different signatures. When +// compiled to wasm, each unique FFI function paired with signature generates a +// wasm import. +class AsmJSImport { + uint32_t ffiIndex_; + + public: + AsmJSImport() = default; + explicit AsmJSImport(uint32_t ffiIndex) : ffiIndex_(ffiIndex) {} + uint32_t ffiIndex() const { return ffiIndex_; } +}; + +using AsmJSImportVector = Vector; + +// An AsmJSExport logically extends Export with the extra information needed for +// an asm.js exported function, viz., the offsets in module's source chars in +// case the function is toString()ed. +class AsmJSExport { + uint32_t funcIndex_ = 0; + + // All fields are treated as cacheable POD: + uint32_t startOffsetInModule_ = 0; // Store module-start-relative offsets + uint32_t endOffsetInModule_ = 0; // so preserved by serialization. + + public: + AsmJSExport() = default; + AsmJSExport(uint32_t funcIndex, uint32_t startOffsetInModule, + uint32_t endOffsetInModule) + : funcIndex_(funcIndex), + startOffsetInModule_(startOffsetInModule), + endOffsetInModule_(endOffsetInModule) {} + uint32_t funcIndex() const { return funcIndex_; } + uint32_t startOffsetInModule() const { return startOffsetInModule_; } + uint32_t endOffsetInModule() const { return endOffsetInModule_; } +}; + +using AsmJSExportVector = Vector; + +// Holds the immutable guts of an AsmJSModule. +// +// AsmJSMetadata is built incrementally by ModuleValidator and then shared +// immutably between AsmJSModules. + +struct AsmJSMetadataCacheablePod { + uint32_t numFFIs = 0; + uint32_t srcLength = 0; + uint32_t srcLengthWithRightBrace = 0; + + AsmJSMetadataCacheablePod() = default; +}; + +struct js::AsmJSMetadata : Metadata, AsmJSMetadataCacheablePod { + AsmJSGlobalVector asmJSGlobals; + AsmJSImportVector asmJSImports; + AsmJSExportVector asmJSExports; + CacheableCharsVector asmJSFuncNames; + CacheableChars globalArgumentName; + CacheableChars importArgumentName; + CacheableChars bufferArgumentName; + + // These values are not serialized since they are relative to the + // containing script which can be different between serialization and + // deserialization contexts. Thus, they must be set explicitly using the + // ambient Parser/ScriptSource after deserialization. + // + // srcStart refers to the offset in the ScriptSource to the beginning of + // the asm.js module function. If the function has been created with the + // Function constructor, this will be the first character in the function + // source. Otherwise, it will be the opening parenthesis of the arguments + // list. + uint32_t toStringStart; + uint32_t srcStart; + bool strict; + bool shouldResistFingerprinting = false; + RefPtr source; + + uint32_t srcEndBeforeCurly() const { return srcStart + srcLength; } + uint32_t srcEndAfterCurly() const { + return srcStart + srcLengthWithRightBrace; + } + + AsmJSMetadata() + : Metadata(ModuleKind::AsmJS), + toStringStart(0), + srcStart(0), + strict(false) {} + ~AsmJSMetadata() override = default; + + const AsmJSExport& lookupAsmJSExport(uint32_t funcIndex) const { + // The AsmJSExportVector isn't stored in sorted order so do a linear + // search. This is for the super-cold and already-expensive toString() + // path and the number of exports is generally small. + for (const AsmJSExport& exp : asmJSExports) { + if (exp.funcIndex() == funcIndex) { + return exp; + } + } + MOZ_CRASH("missing asm.js func export"); + } + + bool mutedErrors() const override { return source->mutedErrors(); } + const char16_t* displayURL() const override { + return source->hasDisplayURL() ? source->displayURL() : nullptr; + } + ScriptSource* maybeScriptSource() const override { return source.get(); } + bool getFuncName(NameContext ctx, uint32_t funcIndex, + UTF8Bytes* name) const override { + const char* p = asmJSFuncNames[funcIndex].get(); + if (!p) { + return true; + } + return name->append(p, strlen(p)); + } + + AsmJSMetadataCacheablePod& pod() { return *this; } + const AsmJSMetadataCacheablePod& pod() const { return *this; } +}; + +using MutableAsmJSMetadata = RefPtr; + +/*****************************************************************************/ +// ParseNode utilities + +static inline ParseNode* NextNode(ParseNode* pn) { return pn->pn_next; } + +static inline ParseNode* UnaryKid(ParseNode* pn) { + return pn->as().kid(); +} + +static inline ParseNode* BinaryRight(ParseNode* pn) { + return pn->as().right(); +} + +static inline ParseNode* BinaryLeft(ParseNode* pn) { + return pn->as().left(); +} + +static inline ParseNode* ReturnExpr(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::ReturnStmt)); + return UnaryKid(pn); +} + +static inline ParseNode* TernaryKid1(ParseNode* pn) { + return pn->as().kid1(); +} + +static inline ParseNode* TernaryKid2(ParseNode* pn) { + return pn->as().kid2(); +} + +static inline ParseNode* TernaryKid3(ParseNode* pn) { + return pn->as().kid3(); +} + +static inline ParseNode* ListHead(ParseNode* pn) { + return pn->as().head(); +} + +static inline unsigned ListLength(ParseNode* pn) { + return pn->as().count(); +} + +static inline ParseNode* CallCallee(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::CallExpr)); + return BinaryLeft(pn); +} + +static inline unsigned CallArgListLength(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::CallExpr)); + return ListLength(BinaryRight(pn)); +} + +static inline ParseNode* CallArgList(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::CallExpr)); + return ListHead(BinaryRight(pn)); +} + +static inline ParseNode* VarListHead(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::VarStmt) || + pn->isKind(ParseNodeKind::ConstDecl)); + return ListHead(pn); +} + +static inline bool IsDefaultCase(ParseNode* pn) { + return pn->as().isDefault(); +} + +static inline ParseNode* CaseExpr(ParseNode* pn) { + return pn->as().caseExpression(); +} + +static inline ParseNode* CaseBody(ParseNode* pn) { + return pn->as().statementList(); +} + +static inline ParseNode* BinaryOpLeft(ParseNode* pn) { + MOZ_ASSERT(pn->isBinaryOperation()); + MOZ_ASSERT(pn->as().count() == 2); + return ListHead(pn); +} + +static inline ParseNode* BinaryOpRight(ParseNode* pn) { + MOZ_ASSERT(pn->isBinaryOperation()); + MOZ_ASSERT(pn->as().count() == 2); + return NextNode(ListHead(pn)); +} + +static inline ParseNode* BitwiseLeft(ParseNode* pn) { return BinaryOpLeft(pn); } + +static inline ParseNode* BitwiseRight(ParseNode* pn) { + return BinaryOpRight(pn); +} + +static inline ParseNode* MultiplyLeft(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::MulExpr)); + return BinaryOpLeft(pn); +} + +static inline ParseNode* MultiplyRight(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::MulExpr)); + return BinaryOpRight(pn); +} + +static inline ParseNode* AddSubLeft(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::AddExpr) || + pn->isKind(ParseNodeKind::SubExpr)); + return BinaryOpLeft(pn); +} + +static inline ParseNode* AddSubRight(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::AddExpr) || + pn->isKind(ParseNodeKind::SubExpr)); + return BinaryOpRight(pn); +} + +static inline ParseNode* DivOrModLeft(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::DivExpr) || + pn->isKind(ParseNodeKind::ModExpr)); + return BinaryOpLeft(pn); +} + +static inline ParseNode* DivOrModRight(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::DivExpr) || + pn->isKind(ParseNodeKind::ModExpr)); + return BinaryOpRight(pn); +} + +static inline ParseNode* ComparisonLeft(ParseNode* pn) { + return BinaryOpLeft(pn); +} + +static inline ParseNode* ComparisonRight(ParseNode* pn) { + return BinaryOpRight(pn); +} + +static inline bool IsExpressionStatement(ParseNode* pn) { + return pn->isKind(ParseNodeKind::ExpressionStmt); +} + +static inline ParseNode* ExpressionStatementExpr(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::ExpressionStmt)); + return UnaryKid(pn); +} + +static inline TaggedParserAtomIndex LoopControlMaybeLabel(ParseNode* pn) { + MOZ_ASSERT(pn->isKind(ParseNodeKind::BreakStmt) || + pn->isKind(ParseNodeKind::ContinueStmt)); + return pn->as().label(); +} + +static inline TaggedParserAtomIndex LabeledStatementLabel(ParseNode* pn) { + return pn->as().label(); +} + +static inline ParseNode* LabeledStatementStatement(ParseNode* pn) { + return pn->as().statement(); +} + +static double NumberNodeValue(ParseNode* pn) { + return pn->as().value(); +} + +static bool NumberNodeHasFrac(ParseNode* pn) { + return pn->as().decimalPoint() == HasDecimal; +} + +static ParseNode* DotBase(ParseNode* pn) { + return &pn->as().expression(); +} + +static TaggedParserAtomIndex DotMember(ParseNode* pn) { + return pn->as().name(); +} + +static ParseNode* ElemBase(ParseNode* pn) { + return &pn->as().expression(); +} + +static ParseNode* ElemIndex(ParseNode* pn) { + return &pn->as().key(); +} + +static inline TaggedParserAtomIndex FunctionName(FunctionNode* funNode) { + if (auto name = funNode->funbox()->explicitName()) { + return name; + } + return TaggedParserAtomIndex::null(); +} + +static inline ParseNode* FunctionFormalParametersList(FunctionNode* fn, + unsigned* numFormals) { + ParamsBodyNode* argsBody = fn->body(); + + // The number of formals is equal to the number of parameters (excluding the + // trailing lexical scope). There are no destructuring or rest parameters for + // asm.js functions. + *numFormals = argsBody->count(); + + // If the function has been fully parsed, the trailing function body node is a + // lexical scope. If we've only parsed the function parameters, the last node + // is the last parameter. + if (*numFormals > 0 && argsBody->last()->is()) { + MOZ_ASSERT(argsBody->last()->as().scopeBody()->isKind( + ParseNodeKind::StatementList)); + (*numFormals)--; + } + + return argsBody->head(); +} + +static inline ParseNode* FunctionStatementList(FunctionNode* funNode) { + LexicalScopeNode* last = funNode->body()->body(); + MOZ_ASSERT(last->isEmptyScope()); + ParseNode* body = last->scopeBody(); + MOZ_ASSERT(body->isKind(ParseNodeKind::StatementList)); + return body; +} + +static inline bool IsNormalObjectField(ParseNode* pn) { + return pn->isKind(ParseNodeKind::PropertyDefinition) && + pn->as().accessorType() == AccessorType::None && + BinaryLeft(pn)->isKind(ParseNodeKind::ObjectPropertyName); +} + +static inline TaggedParserAtomIndex ObjectNormalFieldName(ParseNode* pn) { + MOZ_ASSERT(IsNormalObjectField(pn)); + MOZ_ASSERT(BinaryLeft(pn)->isKind(ParseNodeKind::ObjectPropertyName)); + return BinaryLeft(pn)->as().atom(); +} + +static inline ParseNode* ObjectNormalFieldInitializer(ParseNode* pn) { + MOZ_ASSERT(IsNormalObjectField(pn)); + return BinaryRight(pn); +} + +static inline bool IsUseOfName(ParseNode* pn, TaggedParserAtomIndex name) { + return pn->isName(name); +} + +static inline bool IsIgnoredDirectiveName(TaggedParserAtomIndex atom) { + return atom != TaggedParserAtomIndex::WellKnown::useStrict(); +} + +static inline bool IsIgnoredDirective(ParseNode* pn) { + return pn->isKind(ParseNodeKind::ExpressionStmt) && + UnaryKid(pn)->isKind(ParseNodeKind::StringExpr) && + IsIgnoredDirectiveName(UnaryKid(pn)->as().atom()); +} + +static inline bool IsEmptyStatement(ParseNode* pn) { + return pn->isKind(ParseNodeKind::EmptyStmt); +} + +static inline ParseNode* SkipEmptyStatements(ParseNode* pn) { + while (pn && IsEmptyStatement(pn)) { + pn = pn->pn_next; + } + return pn; +} + +static inline ParseNode* NextNonEmptyStatement(ParseNode* pn) { + return SkipEmptyStatements(pn->pn_next); +} + +template +static bool GetToken(AsmJSParser& parser, TokenKind* tkp) { + auto& ts = parser.tokenStream; + TokenKind tk; + while (true) { + if (!ts.getToken(&tk, TokenStreamShared::SlashIsRegExp)) { + return false; + } + if (tk != TokenKind::Semi) { + break; + } + } + *tkp = tk; + return true; +} + +template +static bool PeekToken(AsmJSParser& parser, TokenKind* tkp) { + auto& ts = parser.tokenStream; + TokenKind tk; + while (true) { + if (!ts.peekToken(&tk, TokenStream::SlashIsRegExp)) { + return false; + } + if (tk != TokenKind::Semi) { + break; + } + ts.consumeKnownToken(TokenKind::Semi, TokenStreamShared::SlashIsRegExp); + } + *tkp = tk; + return true; +} + +template +static bool ParseVarOrConstStatement(AsmJSParser& parser, + ParseNode** var) { + TokenKind tk; + if (!PeekToken(parser, &tk)) { + return false; + } + if (tk != TokenKind::Var && tk != TokenKind::Const) { + *var = nullptr; + return true; + } + + *var = parser.statementListItem(YieldIsName); + if (!*var) { + return false; + } + + MOZ_ASSERT((*var)->isKind(ParseNodeKind::VarStmt) || + (*var)->isKind(ParseNodeKind::ConstDecl)); + return true; +} + +/*****************************************************************************/ + +// Represents the type and value of an asm.js numeric literal. +// +// A literal is a double iff the literal contains a decimal point (even if the +// fractional part is 0). Otherwise, integers may be classified: +// fixnum: [0, 2^31) +// negative int: [-2^31, 0) +// big unsigned: [2^31, 2^32) +// out of range: otherwise +// Lastly, a literal may be a float literal which is any double or integer +// literal coerced with Math.fround. +class NumLit { + public: + enum Which { + Fixnum, + NegativeInt, + BigUnsigned, + Double, + Float, + OutOfRangeInt = -1 + }; + + private: + Which which_; + JS::Value value_; + + public: + NumLit() = default; + + NumLit(Which w, const Value& v) : which_(w), value_(v) {} + + Which which() const { return which_; } + + int32_t toInt32() const { + MOZ_ASSERT(which_ == Fixnum || which_ == NegativeInt || + which_ == BigUnsigned); + return value_.toInt32(); + } + + uint32_t toUint32() const { return (uint32_t)toInt32(); } + + double toDouble() const { + MOZ_ASSERT(which_ == Double); + return value_.toDouble(); + } + + float toFloat() const { + MOZ_ASSERT(which_ == Float); + return float(value_.toDouble()); + } + + Value scalarValue() const { + MOZ_ASSERT(which_ != OutOfRangeInt); + return value_; + } + + bool valid() const { return which_ != OutOfRangeInt; } + + bool isZeroBits() const { + MOZ_ASSERT(valid()); + switch (which()) { + case NumLit::Fixnum: + case NumLit::NegativeInt: + case NumLit::BigUnsigned: + return toInt32() == 0; + case NumLit::Double: + return IsPositiveZero(toDouble()); + case NumLit::Float: + return IsPositiveZero(toFloat()); + case NumLit::OutOfRangeInt: + MOZ_CRASH("can't be here because of valid() check above"); + } + return false; + } + + LitValPOD value() const { + switch (which_) { + case NumLit::Fixnum: + case NumLit::NegativeInt: + case NumLit::BigUnsigned: + return LitValPOD(toUint32()); + case NumLit::Float: + return LitValPOD(toFloat()); + case NumLit::Double: + return LitValPOD(toDouble()); + case NumLit::OutOfRangeInt:; + } + MOZ_CRASH("bad literal"); + } +}; + +// Represents the type of a general asm.js expression. +// +// A canonical subset of types representing the coercion targets: Int, Float, +// Double. +// +// Void is also part of the canonical subset. + +class Type { + public: + enum Which { + Fixnum = NumLit::Fixnum, + Signed = NumLit::NegativeInt, + Unsigned = NumLit::BigUnsigned, + DoubleLit = NumLit::Double, + Float = NumLit::Float, + Double, + MaybeDouble, + MaybeFloat, + Floatish, + Int, + Intish, + Void + }; + + private: + Which which_; + + public: + Type() = default; + MOZ_IMPLICIT Type(Which w) : which_(w) {} + + // Map an already canonicalized Type to the return type of a function call. + static Type ret(Type t) { + MOZ_ASSERT(t.isCanonical()); + // The 32-bit external type is Signed, not Int. + return t.isInt() ? Signed : t; + } + + static Type lit(const NumLit& lit) { + MOZ_ASSERT(lit.valid()); + Which which = Type::Which(lit.which()); + MOZ_ASSERT(which >= Fixnum && which <= Float); + Type t; + t.which_ = which; + return t; + } + + // Map |t| to one of the canonical vartype representations of a + // wasm::ValType. + static Type canonicalize(Type t) { + switch (t.which()) { + case Fixnum: + case Signed: + case Unsigned: + case Int: + return Int; + + case Float: + return Float; + + case DoubleLit: + case Double: + return Double; + + case Void: + return Void; + + case MaybeDouble: + case MaybeFloat: + case Floatish: + case Intish: + // These types need some kind of coercion, they can't be mapped + // to an VarType. + break; + } + MOZ_CRASH("Invalid vartype"); + } + + Which which() const { return which_; } + + bool operator==(Type rhs) const { return which_ == rhs.which_; } + bool operator!=(Type rhs) const { return which_ != rhs.which_; } + + bool operator<=(Type rhs) const { + switch (rhs.which_) { + case Signed: + return isSigned(); + case Unsigned: + return isUnsigned(); + case DoubleLit: + return isDoubleLit(); + case Double: + return isDouble(); + case Float: + return isFloat(); + case MaybeDouble: + return isMaybeDouble(); + case MaybeFloat: + return isMaybeFloat(); + case Floatish: + return isFloatish(); + case Int: + return isInt(); + case Intish: + return isIntish(); + case Fixnum: + return isFixnum(); + case Void: + return isVoid(); + } + MOZ_CRASH("unexpected rhs type"); + } + + bool isFixnum() const { return which_ == Fixnum; } + + bool isSigned() const { return which_ == Signed || which_ == Fixnum; } + + bool isUnsigned() const { return which_ == Unsigned || which_ == Fixnum; } + + bool isInt() const { return isSigned() || isUnsigned() || which_ == Int; } + + bool isIntish() const { return isInt() || which_ == Intish; } + + bool isDoubleLit() const { return which_ == DoubleLit; } + + bool isDouble() const { return isDoubleLit() || which_ == Double; } + + bool isMaybeDouble() const { return isDouble() || which_ == MaybeDouble; } + + bool isFloat() const { return which_ == Float; } + + bool isMaybeFloat() const { return isFloat() || which_ == MaybeFloat; } + + bool isFloatish() const { return isMaybeFloat() || which_ == Floatish; } + + bool isVoid() const { return which_ == Void; } + + bool isExtern() const { return isDouble() || isSigned(); } + + // Check if this is one of the valid types for a function argument. + bool isArgType() const { return isInt() || isFloat() || isDouble(); } + + // Check if this is one of the valid types for a function return value. + bool isReturnType() const { + return isSigned() || isFloat() || isDouble() || isVoid(); + } + + // Check if this is one of the valid types for a global variable. + bool isGlobalVarType() const { return isArgType(); } + + // Check if this is one of the canonical vartype representations of a + // wasm::ValType, or is void. See Type::canonicalize(). + bool isCanonical() const { + switch (which()) { + case Int: + case Float: + case Double: + case Void: + return true; + default: + return false; + } + } + + // Check if this is a canonical representation of a wasm::ValType. + bool isCanonicalValType() const { return !isVoid() && isCanonical(); } + + // Convert this canonical type to a wasm::ValType. + ValType canonicalToValType() const { + switch (which()) { + case Int: + return ValType::I32; + case Float: + return ValType::F32; + case Double: + return ValType::F64; + default: + MOZ_CRASH("Need canonical type"); + } + } + + Maybe canonicalToReturnType() const { + return isVoid() ? Nothing() : Some(canonicalToValType()); + } + + // Convert this type to a wasm::TypeCode for use in a wasm + // block signature. This works for all types, including non-canonical + // ones. Consequently, the type isn't valid for subsequent asm.js + // validation; it's only valid for use in producing wasm. + TypeCode toWasmBlockSignatureType() const { + switch (which()) { + case Fixnum: + case Signed: + case Unsigned: + case Int: + case Intish: + return TypeCode::I32; + + case Float: + case MaybeFloat: + case Floatish: + return TypeCode::F32; + + case DoubleLit: + case Double: + case MaybeDouble: + return TypeCode::F64; + + case Void: + return TypeCode::BlockVoid; + } + MOZ_CRASH("Invalid Type"); + } + + const char* toChars() const { + switch (which_) { + case Double: + return "double"; + case DoubleLit: + return "doublelit"; + case MaybeDouble: + return "double?"; + case Float: + return "float"; + case Floatish: + return "floatish"; + case MaybeFloat: + return "float?"; + case Fixnum: + return "fixnum"; + case Int: + return "int"; + case Signed: + return "signed"; + case Unsigned: + return "unsigned"; + case Intish: + return "intish"; + case Void: + return "void"; + } + MOZ_CRASH("Invalid Type"); + } +}; + +static const unsigned VALIDATION_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024; + +class MOZ_STACK_CLASS ModuleValidatorShared { + public: + struct Memory { + MemoryUsage usage; + uint64_t minLength; + + uint64_t minPages() const { return DivideRoundingUp(minLength, PageSize); } + + Memory() = default; + }; + + class Func { + TaggedParserAtomIndex name_; + uint32_t sigIndex_; + uint32_t firstUse_; + uint32_t funcDefIndex_; + + bool defined_; + + // Available when defined: + uint32_t srcBegin_; + uint32_t srcEnd_; + uint32_t line_; + Bytes bytes_; + Uint32Vector callSiteLineNums_; + + public: + Func(TaggedParserAtomIndex name, uint32_t sigIndex, uint32_t firstUse, + uint32_t funcDefIndex) + : name_(name), + sigIndex_(sigIndex), + firstUse_(firstUse), + funcDefIndex_(funcDefIndex), + defined_(false), + srcBegin_(0), + srcEnd_(0), + line_(0) {} + + TaggedParserAtomIndex name() const { return name_; } + uint32_t sigIndex() const { return sigIndex_; } + uint32_t firstUse() const { return firstUse_; } + bool defined() const { return defined_; } + uint32_t funcDefIndex() const { return funcDefIndex_; } + + void define(ParseNode* fn, uint32_t line, Bytes&& bytes, + Uint32Vector&& callSiteLineNums) { + MOZ_ASSERT(!defined_); + defined_ = true; + srcBegin_ = fn->pn_pos.begin; + srcEnd_ = fn->pn_pos.end; + line_ = line; + bytes_ = std::move(bytes); + callSiteLineNums_ = std::move(callSiteLineNums); + } + + uint32_t srcBegin() const { + MOZ_ASSERT(defined_); + return srcBegin_; + } + uint32_t srcEnd() const { + MOZ_ASSERT(defined_); + return srcEnd_; + } + uint32_t line() const { + MOZ_ASSERT(defined_); + return line_; + } + const Bytes& bytes() const { + MOZ_ASSERT(defined_); + return bytes_; + } + Uint32Vector& callSiteLineNums() { + MOZ_ASSERT(defined_); + return callSiteLineNums_; + } + }; + + using ConstFuncVector = Vector; + using FuncVector = Vector; + + class Table { + uint32_t sigIndex_; + TaggedParserAtomIndex name_; + uint32_t firstUse_; + uint32_t mask_; + bool defined_; + + public: + Table(uint32_t sigIndex, TaggedParserAtomIndex name, uint32_t firstUse, + uint32_t mask) + : sigIndex_(sigIndex), + name_(name), + firstUse_(firstUse), + mask_(mask), + defined_(false) {} + + Table(Table&& rhs) = delete; + + uint32_t sigIndex() const { return sigIndex_; } + TaggedParserAtomIndex name() const { return name_; } + uint32_t firstUse() const { return firstUse_; } + unsigned mask() const { return mask_; } + bool defined() const { return defined_; } + void define() { + MOZ_ASSERT(!defined_); + defined_ = true; + } + }; + + using TableVector = Vector; + + class Global { + public: + enum Which { + Variable, + ConstantLiteral, + ConstantImport, + Function, + Table, + FFI, + ArrayView, + ArrayViewCtor, + MathBuiltinFunction + }; + + private: + Which which_; + union U { + struct VarOrConst { + Type::Which type_; + unsigned index_; + NumLit literalValue_; + + VarOrConst(unsigned index, const NumLit& lit) + : type_(Type::lit(lit).which()), + index_(index), + literalValue_(lit) // copies |lit| + {} + + VarOrConst(unsigned index, Type::Which which) + : type_(which), index_(index) { + // The |literalValue_| field remains unused and + // uninitialized for non-constant variables. + } + + explicit VarOrConst(double constant) + : type_(Type::Double), + literalValue_(NumLit::Double, DoubleValue(constant)) { + // The index_ field is unused and uninitialized for + // constant doubles. + } + } varOrConst; + uint32_t funcDefIndex_; + uint32_t tableIndex_; + uint32_t ffiIndex_; + Scalar::Type viewType_; + AsmJSMathBuiltinFunction mathBuiltinFunc_; + + // |varOrConst|, through |varOrConst.literalValue_|, has a + // non-trivial constructor and therefore MUST be placement-new'd + // into existence. + MOZ_PUSH_DISABLE_NONTRIVIAL_UNION_WARNINGS + U() : funcDefIndex_(0) {} + MOZ_POP_DISABLE_NONTRIVIAL_UNION_WARNINGS + } u; + + friend class ModuleValidatorShared; + template + friend class ModuleValidator; + friend class js::LifoAlloc; + + explicit Global(Which which) : which_(which) {} + + public: + Which which() const { return which_; } + Type varOrConstType() const { + MOZ_ASSERT(which_ == Variable || which_ == ConstantLiteral || + which_ == ConstantImport); + return u.varOrConst.type_; + } + unsigned varOrConstIndex() const { + MOZ_ASSERT(which_ == Variable || which_ == ConstantImport); + return u.varOrConst.index_; + } + bool isConst() const { + return which_ == ConstantLiteral || which_ == ConstantImport; + } + NumLit constLiteralValue() const { + MOZ_ASSERT(which_ == ConstantLiteral); + return u.varOrConst.literalValue_; + } + uint32_t funcDefIndex() const { + MOZ_ASSERT(which_ == Function); + return u.funcDefIndex_; + } + uint32_t tableIndex() const { + MOZ_ASSERT(which_ == Table); + return u.tableIndex_; + } + unsigned ffiIndex() const { + MOZ_ASSERT(which_ == FFI); + return u.ffiIndex_; + } + Scalar::Type viewType() const { + MOZ_ASSERT(which_ == ArrayView || which_ == ArrayViewCtor); + return u.viewType_; + } + bool isMathFunction() const { return which_ == MathBuiltinFunction; } + AsmJSMathBuiltinFunction mathBuiltinFunction() const { + MOZ_ASSERT(which_ == MathBuiltinFunction); + return u.mathBuiltinFunc_; + } + }; + + struct MathBuiltin { + enum Kind { Function, Constant }; + Kind kind; + + union { + double cst; + AsmJSMathBuiltinFunction func; + } u; + + MathBuiltin() : kind(Kind(-1)), u{} {} + explicit MathBuiltin(double cst) : kind(Constant) { u.cst = cst; } + explicit MathBuiltin(AsmJSMathBuiltinFunction func) : kind(Function) { + u.func = func; + } + }; + + struct ArrayView { + ArrayView(TaggedParserAtomIndex name, Scalar::Type type) + : name(name), type(type) {} + + TaggedParserAtomIndex name; + Scalar::Type type; + }; + + protected: + class HashableSig { + uint32_t sigIndex_; + const TypeContext& types_; + + public: + HashableSig(uint32_t sigIndex, const TypeContext& types) + : sigIndex_(sigIndex), types_(types) {} + uint32_t sigIndex() const { return sigIndex_; } + const FuncType& funcType() const { return types_[sigIndex_].funcType(); } + + // Implement HashPolicy: + using Lookup = const FuncType&; + static HashNumber hash(Lookup l) { return l.hash(nullptr); } + static bool match(HashableSig lhs, Lookup rhs) { + return FuncType::strictlyEquals(lhs.funcType(), rhs); + } + }; + + class NamedSig : public HashableSig { + TaggedParserAtomIndex name_; + + public: + NamedSig(TaggedParserAtomIndex name, uint32_t sigIndex, + const TypeContext& types) + : HashableSig(sigIndex, types), name_(name) {} + TaggedParserAtomIndex name() const { return name_; } + + // Implement HashPolicy: + struct Lookup { + TaggedParserAtomIndex name; + const FuncType& funcType; + Lookup(TaggedParserAtomIndex name, const FuncType& funcType) + : name(name), funcType(funcType) {} + }; + static HashNumber hash(Lookup l) { + return HashGeneric(TaggedParserAtomIndexHasher::hash(l.name), + l.funcType.hash(nullptr)); + } + static bool match(NamedSig lhs, Lookup rhs) { + return lhs.name() == rhs.name && + FuncType::strictlyEquals(lhs.funcType(), rhs.funcType); + } + }; + + using SigSet = HashSet; + using FuncImportMap = HashMap; + using GlobalMap = + HashMap; + using MathNameMap = + HashMap; + using ArrayViewVector = Vector; + + protected: + FrontendContext* fc_; + ParserAtomsTable& parserAtoms_; + FunctionNode* moduleFunctionNode_; + TaggedParserAtomIndex moduleFunctionName_; + TaggedParserAtomIndex globalArgumentName_; + TaggedParserAtomIndex importArgumentName_; + TaggedParserAtomIndex bufferArgumentName_; + MathNameMap standardLibraryMathNames_; + + // Validation-internal state: + LifoAlloc validationLifo_; + Memory memory_; + FuncVector funcDefs_; + TableVector tables_; + GlobalMap globalMap_; + SigSet sigSet_; + FuncImportMap funcImportMap_; + ArrayViewVector arrayViews_; + + // State used to build the AsmJSModule in finish(): + CompilerEnvironment compilerEnv_; + ModuleEnvironment moduleEnv_; + MutableAsmJSMetadata asmJSMetadata_; + + // Error reporting: + UniqueChars errorString_ = nullptr; + uint32_t errorOffset_ = UINT32_MAX; + bool errorOverRecursed_ = false; + + protected: + ModuleValidatorShared(FrontendContext* fc, ParserAtomsTable& parserAtoms, + FunctionNode* moduleFunctionNode) + : fc_(fc), + parserAtoms_(parserAtoms), + moduleFunctionNode_(moduleFunctionNode), + moduleFunctionName_(FunctionName(moduleFunctionNode)), + standardLibraryMathNames_(fc), + validationLifo_(VALIDATION_LIFO_DEFAULT_CHUNK_SIZE), + funcDefs_(fc), + tables_(fc), + globalMap_(fc), + sigSet_(fc), + funcImportMap_(fc), + arrayViews_(fc), + compilerEnv_(CompileMode::Once, Tier::Optimized, DebugEnabled::False), + moduleEnv_(FeatureArgs(), ModuleKind::AsmJS) { + compilerEnv_.computeParameters(); + memory_.minLength = RoundUpToNextValidAsmJSHeapLength(0); + } + + protected: + [[nodiscard]] bool initModuleEnvironment() { return moduleEnv_.init(); } + + [[nodiscard]] bool addStandardLibraryMathInfo() { + static constexpr struct { + const char* name; + AsmJSMathBuiltinFunction func; + } functions[] = { + {"sin", AsmJSMathBuiltin_sin}, {"cos", AsmJSMathBuiltin_cos}, + {"tan", AsmJSMathBuiltin_tan}, {"asin", AsmJSMathBuiltin_asin}, + {"acos", AsmJSMathBuiltin_acos}, {"atan", AsmJSMathBuiltin_atan}, + {"ceil", AsmJSMathBuiltin_ceil}, {"floor", AsmJSMathBuiltin_floor}, + {"exp", AsmJSMathBuiltin_exp}, {"log", AsmJSMathBuiltin_log}, + {"pow", AsmJSMathBuiltin_pow}, {"sqrt", AsmJSMathBuiltin_sqrt}, + {"abs", AsmJSMathBuiltin_abs}, {"atan2", AsmJSMathBuiltin_atan2}, + {"imul", AsmJSMathBuiltin_imul}, {"clz32", AsmJSMathBuiltin_clz32}, + {"fround", AsmJSMathBuiltin_fround}, {"min", AsmJSMathBuiltin_min}, + {"max", AsmJSMathBuiltin_max}, + }; + + auto AddMathFunction = [this](const char* name, + AsmJSMathBuiltinFunction func) { + auto atom = parserAtoms_.internAscii(fc_, name, strlen(name)); + if (!atom) { + return false; + } + MathBuiltin builtin(func); + return this->standardLibraryMathNames_.putNew(atom, builtin); + }; + + for (const auto& info : functions) { + if (!AddMathFunction(info.name, info.func)) { + return false; + } + } + + static constexpr struct { + const char* name; + double value; + } constants[] = { + {"E", M_E}, + {"LN10", M_LN10}, + {"LN2", M_LN2}, + {"LOG2E", M_LOG2E}, + {"LOG10E", M_LOG10E}, + {"PI", M_PI}, + {"SQRT1_2", M_SQRT1_2}, + {"SQRT2", M_SQRT2}, + }; + + auto AddMathConstant = [this](const char* name, double cst) { + auto atom = parserAtoms_.internAscii(fc_, name, strlen(name)); + if (!atom) { + return false; + } + MathBuiltin builtin(cst); + return this->standardLibraryMathNames_.putNew(atom, builtin); + }; + + for (const auto& info : constants) { + if (!AddMathConstant(info.name, info.value)) { + return false; + } + } + + return true; + } + + public: + FrontendContext* fc() const { return fc_; } + TaggedParserAtomIndex moduleFunctionName() const { + return moduleFunctionName_; + } + TaggedParserAtomIndex globalArgumentName() const { + return globalArgumentName_; + } + TaggedParserAtomIndex importArgumentName() const { + return importArgumentName_; + } + TaggedParserAtomIndex bufferArgumentName() const { + return bufferArgumentName_; + } + const ModuleEnvironment& env() { return moduleEnv_; } + + void initModuleFunctionName(TaggedParserAtomIndex name) { + MOZ_ASSERT(!moduleFunctionName_); + moduleFunctionName_ = name; + } + [[nodiscard]] bool initGlobalArgumentName(TaggedParserAtomIndex n) { + globalArgumentName_ = n; + if (n) { + asmJSMetadata_->globalArgumentName = parserAtoms_.toNewUTF8CharsZ(fc_, n); + if (!asmJSMetadata_->globalArgumentName) { + return false; + } + } + return true; + } + [[nodiscard]] bool initImportArgumentName(TaggedParserAtomIndex n) { + importArgumentName_ = n; + if (n) { + asmJSMetadata_->importArgumentName = parserAtoms_.toNewUTF8CharsZ(fc_, n); + if (!asmJSMetadata_->importArgumentName) { + return false; + } + } + return true; + } + [[nodiscard]] bool initBufferArgumentName(TaggedParserAtomIndex n) { + bufferArgumentName_ = n; + if (n) { + asmJSMetadata_->bufferArgumentName = parserAtoms_.toNewUTF8CharsZ(fc_, n); + if (!asmJSMetadata_->bufferArgumentName) { + return false; + } + } + return true; + } + bool addGlobalVarInit(TaggedParserAtomIndex var, const NumLit& lit, Type type, + bool isConst) { + MOZ_ASSERT(type.isGlobalVarType()); + MOZ_ASSERT(type == Type::canonicalize(Type::lit(lit))); + + uint32_t index = moduleEnv_.globals.length(); + if (!moduleEnv_.globals.emplaceBack(type.canonicalToValType(), !isConst, + index, ModuleKind::AsmJS)) { + return false; + } + + Global::Which which = isConst ? Global::ConstantLiteral : Global::Variable; + Global* global = validationLifo_.new_(which); + if (!global) { + return false; + } + if (isConst) { + new (&global->u.varOrConst) Global::U::VarOrConst(index, lit); + } else { + new (&global->u.varOrConst) Global::U::VarOrConst(index, type.which()); + } + if (!globalMap_.putNew(var, global)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::Variable, nullptr); + g.pod.u.var.initKind_ = AsmJSGlobal::InitConstant; + g.pod.u.var.u.val_ = lit.value(); + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addGlobalVarImport(TaggedParserAtomIndex var, + TaggedParserAtomIndex field, Type type, + bool isConst) { + MOZ_ASSERT(type.isGlobalVarType()); + + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, field); + if (!fieldChars) { + return false; + } + + uint32_t index = moduleEnv_.globals.length(); + ValType valType = type.canonicalToValType(); + if (!moduleEnv_.globals.emplaceBack(valType, !isConst, index, + ModuleKind::AsmJS)) { + return false; + } + + Global::Which which = isConst ? Global::ConstantImport : Global::Variable; + Global* global = validationLifo_.new_(which); + if (!global) { + return false; + } + new (&global->u.varOrConst) Global::U::VarOrConst(index, type.which()); + if (!globalMap_.putNew(var, global)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::Variable, std::move(fieldChars)); + g.pod.u.var.initKind_ = AsmJSGlobal::InitImport; + g.pod.u.var.u.importValType_ = valType.packed(); + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addArrayView(TaggedParserAtomIndex var, Scalar::Type vt, + TaggedParserAtomIndex maybeField) { + UniqueChars fieldChars; + if (maybeField) { + fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, maybeField); + if (!fieldChars) { + return false; + } + } + + if (!arrayViews_.append(ArrayView(var, vt))) { + return false; + } + + Global* global = validationLifo_.new_(Global::ArrayView); + if (!global) { + return false; + } + new (&global->u.viewType_) Scalar::Type(vt); + if (!globalMap_.putNew(var, global)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::ArrayView, std::move(fieldChars)); + g.pod.u.viewType_ = vt; + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addMathBuiltinFunction(TaggedParserAtomIndex var, + AsmJSMathBuiltinFunction func, + TaggedParserAtomIndex field) { + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, field); + if (!fieldChars) { + return false; + } + + Global* global = validationLifo_.new_(Global::MathBuiltinFunction); + if (!global) { + return false; + } + new (&global->u.mathBuiltinFunc_) AsmJSMathBuiltinFunction(func); + if (!globalMap_.putNew(var, global)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::MathBuiltinFunction, std::move(fieldChars)); + g.pod.u.mathBuiltinFunc_ = func; + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + + private: + bool addGlobalDoubleConstant(TaggedParserAtomIndex var, double constant) { + Global* global = validationLifo_.new_(Global::ConstantLiteral); + if (!global) { + return false; + } + new (&global->u.varOrConst) Global::U::VarOrConst(constant); + return globalMap_.putNew(var, global); + } + + public: + bool addMathBuiltinConstant(TaggedParserAtomIndex var, double constant, + TaggedParserAtomIndex field) { + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, field); + if (!fieldChars) { + return false; + } + + if (!addGlobalDoubleConstant(var, constant)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::Constant, std::move(fieldChars)); + g.pod.u.constant.value_ = constant; + g.pod.u.constant.kind_ = AsmJSGlobal::MathConstant; + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addGlobalConstant(TaggedParserAtomIndex var, double constant, + TaggedParserAtomIndex field) { + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, field); + if (!fieldChars) { + return false; + } + + if (!addGlobalDoubleConstant(var, constant)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::Constant, std::move(fieldChars)); + g.pod.u.constant.value_ = constant; + g.pod.u.constant.kind_ = AsmJSGlobal::GlobalConstant; + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addArrayViewCtor(TaggedParserAtomIndex var, Scalar::Type vt, + TaggedParserAtomIndex field) { + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, field); + if (!fieldChars) { + return false; + } + + Global* global = validationLifo_.new_(Global::ArrayViewCtor); + if (!global) { + return false; + } + new (&global->u.viewType_) Scalar::Type(vt); + if (!globalMap_.putNew(var, global)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::ArrayViewCtor, std::move(fieldChars)); + g.pod.u.viewType_ = vt; + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addFFI(TaggedParserAtomIndex var, TaggedParserAtomIndex field) { + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, field); + if (!fieldChars) { + return false; + } + + if (asmJSMetadata_->numFFIs == UINT32_MAX) { + return false; + } + uint32_t ffiIndex = asmJSMetadata_->numFFIs++; + + Global* global = validationLifo_.new_(Global::FFI); + if (!global) { + return false; + } + new (&global->u.ffiIndex_) uint32_t(ffiIndex); + if (!globalMap_.putNew(var, global)) { + return false; + } + + AsmJSGlobal g(AsmJSGlobal::FFI, std::move(fieldChars)); + g.pod.u.ffiIndex_ = ffiIndex; + return asmJSMetadata_->asmJSGlobals.append(std::move(g)); + } + bool addExportField(const Func& func, TaggedParserAtomIndex maybeField) { + // Record the field name of this export. + CacheableName fieldName; + if (maybeField) { + UniqueChars fieldChars = parserAtoms_.toNewUTF8CharsZ(fc_, maybeField); + if (!fieldChars) { + return false; + } + fieldName = CacheableName::fromUTF8Chars(std::move(fieldChars)); + } + + // Declare which function is exported which gives us an index into the + // module ExportVector. + uint32_t funcIndex = funcImportMap_.count() + func.funcDefIndex(); + if (!moduleEnv_.exports.emplaceBack(std::move(fieldName), funcIndex, + DefinitionKind::Function)) { + return false; + } + + // The exported function might have already been exported in which case + // the index will refer into the range of AsmJSExports. + return asmJSMetadata_->asmJSExports.emplaceBack( + funcIndex, func.srcBegin() - asmJSMetadata_->srcStart, + func.srcEnd() - asmJSMetadata_->srcStart); + } + + bool defineFuncPtrTable(uint32_t tableIndex, Uint32Vector&& elems) { + Table& table = *tables_[tableIndex]; + if (table.defined()) { + return false; + } + + table.define(); + + for (uint32_t& index : elems) { + index += funcImportMap_.count(); + } + + MutableElemSegment seg = js_new(); + if (!seg) { + return false; + } + seg->elemType = RefType::func(); + seg->tableIndex = tableIndex; + seg->offsetIfActive = Some(InitExpr(LitVal(uint32_t(0)))); + seg->elemFuncIndices = std::move(elems); + return moduleEnv_.elemSegments.append(std::move(seg)); + } + + bool tryConstantAccess(uint64_t start, uint64_t width) { + MOZ_ASSERT(UINT64_MAX - start > width); + uint64_t len = start + width; + if (len > uint64_t(INT32_MAX) + 1) { + return false; + } + len = RoundUpToNextValidAsmJSHeapLength(len); + if (len > memory_.minLength) { + memory_.minLength = len; + } + return true; + } + + // Error handling. + bool hasAlreadyFailed() const { return !!errorString_; } + + bool failOffset(uint32_t offset, const char* str) { + MOZ_ASSERT(!hasAlreadyFailed()); + MOZ_ASSERT(errorOffset_ == UINT32_MAX); + MOZ_ASSERT(str); + errorOffset_ = offset; + errorString_ = DuplicateString(str); + return false; + } + + bool fail(ParseNode* pn, const char* str) { + return failOffset(pn->pn_pos.begin, str); + } + + bool failfVAOffset(uint32_t offset, const char* fmt, va_list ap) + MOZ_FORMAT_PRINTF(3, 0) { + MOZ_ASSERT(!hasAlreadyFailed()); + MOZ_ASSERT(errorOffset_ == UINT32_MAX); + MOZ_ASSERT(fmt); + errorOffset_ = offset; + errorString_ = JS_vsmprintf(fmt, ap); + return false; + } + + bool failfOffset(uint32_t offset, const char* fmt, ...) + MOZ_FORMAT_PRINTF(3, 4) { + va_list ap; + va_start(ap, fmt); + failfVAOffset(offset, fmt, ap); + va_end(ap); + return false; + } + + bool failf(ParseNode* pn, const char* fmt, ...) MOZ_FORMAT_PRINTF(3, 4) { + va_list ap; + va_start(ap, fmt); + failfVAOffset(pn->pn_pos.begin, fmt, ap); + va_end(ap); + return false; + } + + bool failNameOffset(uint32_t offset, const char* fmt, + TaggedParserAtomIndex name) { + // This function is invoked without the caller properly rooting its locals. + if (UniqueChars bytes = parserAtoms_.toPrintableString(name)) { + failfOffset(offset, fmt, bytes.get()); + } else { + ReportOutOfMemory(fc_); + } + return false; + } + + bool failName(ParseNode* pn, const char* fmt, TaggedParserAtomIndex name) { + return failNameOffset(pn->pn_pos.begin, fmt, name); + } + + bool failOverRecursed() { + errorOverRecursed_ = true; + return false; + } + + unsigned numArrayViews() const { return arrayViews_.length(); } + const ArrayView& arrayView(unsigned i) const { return arrayViews_[i]; } + unsigned numFuncDefs() const { return funcDefs_.length(); } + const Func& funcDef(unsigned i) const { return funcDefs_[i]; } + unsigned numFuncPtrTables() const { return tables_.length(); } + Table& table(unsigned i) const { return *tables_[i]; } + + const Global* lookupGlobal(TaggedParserAtomIndex name) const { + if (GlobalMap::Ptr p = globalMap_.lookup(name)) { + return p->value(); + } + return nullptr; + } + + Func* lookupFuncDef(TaggedParserAtomIndex name) { + if (GlobalMap::Ptr p = globalMap_.lookup(name)) { + Global* value = p->value(); + if (value->which() == Global::Function) { + return &funcDefs_[value->funcDefIndex()]; + } + } + return nullptr; + } + + bool lookupStandardLibraryMathName(TaggedParserAtomIndex name, + MathBuiltin* mathBuiltin) const { + if (MathNameMap::Ptr p = standardLibraryMathNames_.lookup(name)) { + *mathBuiltin = p->value(); + return true; + } + return false; + } + + bool startFunctionBodies() { + if (!arrayViews_.empty()) { + memory_.usage = MemoryUsage::Unshared; + } else { + memory_.usage = MemoryUsage::None; + } + return true; + } +}; + +// The ModuleValidator encapsulates the entire validation of an asm.js module. +// Its lifetime goes from the validation of the top components of an asm.js +// module (all the globals), the emission of bytecode for all the functions in +// the module and the validation of function's pointer tables. It also finishes +// the compilation of all the module's stubs. +template +class MOZ_STACK_CLASS ModuleValidator : public ModuleValidatorShared { + private: + AsmJSParser& parser_; + + public: + ModuleValidator(FrontendContext* fc, ParserAtomsTable& parserAtoms, + AsmJSParser& parser, FunctionNode* moduleFunctionNode) + : ModuleValidatorShared(fc, parserAtoms, moduleFunctionNode), + parser_(parser) {} + + ~ModuleValidator() { + if (errorString_) { + MOZ_ASSERT(errorOffset_ != UINT32_MAX); + typeFailure(errorOffset_, errorString_.get()); + } + if (errorOverRecursed_) { + ReportOverRecursed(fc_); + } + } + + private: + // Helpers: + bool newSig(FuncType&& sig, uint32_t* sigIndex) { + if (moduleEnv_.types->length() >= MaxTypes) { + return failCurrentOffset("too many signatures"); + } + + *sigIndex = moduleEnv_.types->length(); + return moduleEnv_.types->addType(std::move(sig)); + } + bool declareSig(FuncType&& sig, uint32_t* sigIndex) { + SigSet::AddPtr p = sigSet_.lookupForAdd(sig); + if (p) { + *sigIndex = p->sigIndex(); + MOZ_ASSERT(FuncType::strictlyEquals( + moduleEnv_.types->type(*sigIndex).funcType(), sig)); + return true; + } + + return newSig(std::move(sig), sigIndex) && + sigSet_.add(p, HashableSig(*sigIndex, *moduleEnv_.types)); + } + + private: + void typeFailure(uint32_t offset, ...) { + va_list args; + va_start(args, offset); + + auto& ts = tokenStream(); + ErrorMetadata metadata; + if (ts.computeErrorMetadata(&metadata, AsVariant(offset))) { + if (ts.anyCharsAccess().options().throwOnAsmJSValidationFailureOption) { + ReportCompileErrorLatin1(fc_, std::move(metadata), nullptr, + JSMSG_USE_ASM_TYPE_FAIL, &args); + } else { + // asm.js type failure is indicated by calling one of the fail* + // functions below. These functions always return false to + // halt asm.js parsing. Whether normal parsing is attempted as + // fallback, depends whether an exception is also set. + // + // If warning succeeds, no exception is set. If warning fails, + // an exception is set and execution will halt. Thus it's safe + // and correct to ignore the return value here. + (void)ts.compileWarning(std::move(metadata), nullptr, + JSMSG_USE_ASM_TYPE_FAIL, &args); + } + } + + va_end(args); + } + + public: + bool init() { + asmJSMetadata_ = js_new(); + if (!asmJSMetadata_) { + ReportOutOfMemory(fc_); + return false; + } + + asmJSMetadata_->toStringStart = + moduleFunctionNode_->funbox()->extent().toStringStart; + asmJSMetadata_->srcStart = moduleFunctionNode_->body()->pn_pos.begin; + asmJSMetadata_->strict = parser_.pc_->sc()->strict() && + !parser_.pc_->sc()->hasExplicitUseStrict(); + asmJSMetadata_->shouldResistFingerprinting = + parser_.options().shouldResistFingerprinting(); + asmJSMetadata_->source = do_AddRef(parser_.ss); + + if (!initModuleEnvironment()) { + return false; + } + return addStandardLibraryMathInfo(); + } + + AsmJSParser& parser() const { return parser_; } + + auto& tokenStream() const { return parser_.tokenStream; } + + bool shouldResistFingerprinting() const { + return asmJSMetadata_->shouldResistFingerprinting; + } + + public: + bool addFuncDef(TaggedParserAtomIndex name, uint32_t firstUse, FuncType&& sig, + Func** func) { + uint32_t sigIndex; + if (!declareSig(std::move(sig), &sigIndex)) { + return false; + } + + uint32_t funcDefIndex = funcDefs_.length(); + if (funcDefIndex >= MaxFuncs) { + return failCurrentOffset("too many functions"); + } + + Global* global = validationLifo_.new_(Global::Function); + if (!global) { + return false; + } + new (&global->u.funcDefIndex_) uint32_t(funcDefIndex); + if (!globalMap_.putNew(name, global)) { + return false; + } + if (!funcDefs_.emplaceBack(name, sigIndex, firstUse, funcDefIndex)) { + return false; + } + *func = &funcDefs_.back(); + return true; + } + bool declareFuncPtrTable(FuncType&& sig, TaggedParserAtomIndex name, + uint32_t firstUse, uint32_t mask, + uint32_t* tableIndex) { + if (mask > MaxTableLength) { + return failCurrentOffset("function pointer table too big"); + } + + MOZ_ASSERT(moduleEnv_.tables.length() == tables_.length()); + *tableIndex = moduleEnv_.tables.length(); + + uint32_t sigIndex; + if (!newSig(std::move(sig), &sigIndex)) { + return false; + } + + MOZ_ASSERT(sigIndex >= moduleEnv_.asmJSSigToTableIndex.length()); + if (!moduleEnv_.asmJSSigToTableIndex.resize(sigIndex + 1)) { + return false; + } + + moduleEnv_.asmJSSigToTableIndex[sigIndex] = moduleEnv_.tables.length(); + if (!moduleEnv_.tables.emplaceBack(RefType::func(), mask + 1, Nothing(), + /* initExpr */ Nothing(), + /*isAsmJS*/ true)) { + return false; + } + + Global* global = validationLifo_.new_(Global::Table); + if (!global) { + return false; + } + + new (&global->u.tableIndex_) uint32_t(*tableIndex); + if (!globalMap_.putNew(name, global)) { + return false; + } + + Table* t = validationLifo_.new_(sigIndex, name, firstUse, mask); + return t && tables_.append(t); + } + bool declareImport(TaggedParserAtomIndex name, FuncType&& sig, + unsigned ffiIndex, uint32_t* importIndex) { + FuncImportMap::AddPtr p = + funcImportMap_.lookupForAdd(NamedSig::Lookup(name, sig)); + if (p) { + *importIndex = p->value(); + return true; + } + + *importIndex = funcImportMap_.count(); + MOZ_ASSERT(*importIndex == asmJSMetadata_->asmJSImports.length()); + + if (*importIndex >= MaxImports) { + return failCurrentOffset("too many imports"); + } + + if (!asmJSMetadata_->asmJSImports.emplaceBack(ffiIndex)) { + return false; + } + + uint32_t sigIndex; + if (!declareSig(std::move(sig), &sigIndex)) { + return false; + } + + return funcImportMap_.add(p, NamedSig(name, sigIndex, *moduleEnv_.types), + *importIndex); + } + + // Error handling. + bool failCurrentOffset(const char* str) { + return failOffset(tokenStream().anyCharsAccess().currentToken().pos.begin, + str); + } + + SharedModule finish() { + MOZ_ASSERT(!moduleEnv_.usesMemory()); + if (memory_.usage != MemoryUsage::None) { + Limits limits; + limits.shared = memory_.usage == MemoryUsage::Shared ? Shareable::True + : Shareable::False; + limits.initial = memory_.minPages(); + limits.maximum = Nothing(); + limits.indexType = IndexType::I32; + moduleEnv_.memory = Some(MemoryDesc(limits)); + } + MOZ_ASSERT(moduleEnv_.funcs.empty()); + if (!moduleEnv_.funcs.resize(funcImportMap_.count() + funcDefs_.length())) { + return nullptr; + } + for (FuncImportMap::Range r = funcImportMap_.all(); !r.empty(); + r.popFront()) { + uint32_t funcIndex = r.front().value(); + uint32_t funcTypeIndex = r.front().key().sigIndex(); + MOZ_ASSERT(!moduleEnv_.funcs[funcIndex].type); + moduleEnv_.funcs[funcIndex] = FuncDesc( + &moduleEnv_.types->type(funcTypeIndex).funcType(), funcTypeIndex); + } + for (const Func& func : funcDefs_) { + uint32_t funcIndex = funcImportMap_.count() + func.funcDefIndex(); + uint32_t funcTypeIndex = func.sigIndex(); + MOZ_ASSERT(!moduleEnv_.funcs[funcIndex].type); + moduleEnv_.funcs[funcIndex] = FuncDesc( + &moduleEnv_.types->type(funcTypeIndex).funcType(), funcTypeIndex); + } + for (const Export& exp : moduleEnv_.exports) { + if (exp.kind() != DefinitionKind::Function) { + continue; + } + uint32_t funcIndex = exp.funcIndex(); + moduleEnv_.declareFuncExported(funcIndex, /* eager */ true, + /* canRefFunc */ false); + } + + moduleEnv_.numFuncImports = funcImportMap_.count(); + + MOZ_ASSERT(asmJSMetadata_->asmJSFuncNames.empty()); + if (!asmJSMetadata_->asmJSFuncNames.resize(funcImportMap_.count())) { + return nullptr; + } + for (const Func& func : funcDefs_) { + CacheableChars funcName = parserAtoms_.toNewUTF8CharsZ(fc_, func.name()); + if (!funcName || + !asmJSMetadata_->asmJSFuncNames.emplaceBack(std::move(funcName))) { + return nullptr; + } + } + + uint32_t endBeforeCurly = + tokenStream().anyCharsAccess().currentToken().pos.end; + asmJSMetadata_->srcLength = endBeforeCurly - asmJSMetadata_->srcStart; + + TokenPos pos; + MOZ_ALWAYS_TRUE( + tokenStream().peekTokenPos(&pos, TokenStreamShared::SlashIsRegExp)); + uint32_t endAfterCurly = pos.end; + asmJSMetadata_->srcLengthWithRightBrace = + endAfterCurly - asmJSMetadata_->srcStart; + + ScriptedCaller scriptedCaller; + if (parser_.ss->filename()) { + scriptedCaller.line = 0; // unused + scriptedCaller.filename = DuplicateString(parser_.ss->filename()); + if (!scriptedCaller.filename) { + return nullptr; + } + } + + // The default options are fine for asm.js + SharedCompileArgs args = + CompileArgs::buildForAsmJS(std::move(scriptedCaller)); + if (!args) { + ReportOutOfMemory(fc_); + return nullptr; + } + + uint32_t codeSectionSize = 0; + for (const Func& func : funcDefs_) { + codeSectionSize += func.bytes().length(); + } + + moduleEnv_.codeSection.emplace(); + moduleEnv_.codeSection->start = 0; + moduleEnv_.codeSection->size = codeSectionSize; + + // asm.js does not have any wasm bytecode to save; view-source is + // provided through the ScriptSource. + SharedBytes bytes = js_new(); + if (!bytes) { + ReportOutOfMemory(fc_); + return nullptr; + } + + ModuleGenerator mg(*args, &moduleEnv_, &compilerEnv_, nullptr, nullptr, + nullptr); + if (!mg.init(asmJSMetadata_.get())) { + return nullptr; + } + + for (Func& func : funcDefs_) { + if (!mg.compileFuncDef(funcImportMap_.count() + func.funcDefIndex(), + func.line(), func.bytes().begin(), + func.bytes().end(), + std::move(func.callSiteLineNums()))) { + return nullptr; + } + } + + if (!mg.finishFuncDefs()) { + return nullptr; + } + + return mg.finishModule(*bytes); + } +}; + +/*****************************************************************************/ +// Numeric literal utilities + +static bool IsNumericNonFloatLiteral(ParseNode* pn) { + // Note: '-' is never rolled into the number; numbers are always positive + // and negations must be applied manually. + return pn->isKind(ParseNodeKind::NumberExpr) || + (pn->isKind(ParseNodeKind::NegExpr) && + UnaryKid(pn)->isKind(ParseNodeKind::NumberExpr)); +} + +static bool IsCallToGlobal(ModuleValidatorShared& m, ParseNode* pn, + const ModuleValidatorShared::Global** global) { + if (!pn->isKind(ParseNodeKind::CallExpr)) { + return false; + } + + ParseNode* callee = CallCallee(pn); + if (!callee->isKind(ParseNodeKind::Name)) { + return false; + } + + *global = m.lookupGlobal(callee->as().name()); + return !!*global; +} + +static bool IsCoercionCall(ModuleValidatorShared& m, ParseNode* pn, + Type* coerceTo, ParseNode** coercedExpr) { + const ModuleValidatorShared::Global* global; + if (!IsCallToGlobal(m, pn, &global)) { + return false; + } + + if (CallArgListLength(pn) != 1) { + return false; + } + + if (coercedExpr) { + *coercedExpr = CallArgList(pn); + } + + if (global->isMathFunction() && + global->mathBuiltinFunction() == AsmJSMathBuiltin_fround) { + *coerceTo = Type::Float; + return true; + } + + return false; +} + +static bool IsFloatLiteral(ModuleValidatorShared& m, ParseNode* pn) { + ParseNode* coercedExpr; + Type coerceTo; + if (!IsCoercionCall(m, pn, &coerceTo, &coercedExpr)) { + return false; + } + // Don't fold into || to avoid clang/memcheck bug (bug 1077031). + if (!coerceTo.isFloat()) { + return false; + } + return IsNumericNonFloatLiteral(coercedExpr); +} + +static bool IsNumericLiteral(ModuleValidatorShared& m, ParseNode* pn) { + return IsNumericNonFloatLiteral(pn) || IsFloatLiteral(m, pn); +} + +// The JS grammar treats -42 as -(42) (i.e., with separate grammar +// productions) for the unary - and literal 42). However, the asm.js spec +// recognizes -42 (modulo parens, so -(42) and -((42))) as a single literal +// so fold the two potential parse nodes into a single double value. +static double ExtractNumericNonFloatValue(ParseNode* pn, + ParseNode** out = nullptr) { + MOZ_ASSERT(IsNumericNonFloatLiteral(pn)); + + if (pn->isKind(ParseNodeKind::NegExpr)) { + pn = UnaryKid(pn); + if (out) { + *out = pn; + } + return -NumberNodeValue(pn); + } + + return NumberNodeValue(pn); +} + +static NumLit ExtractNumericLiteral(ModuleValidatorShared& m, ParseNode* pn) { + MOZ_ASSERT(IsNumericLiteral(m, pn)); + + if (pn->isKind(ParseNodeKind::CallExpr)) { + // Float literals are explicitly coerced and thus the coerced literal may be + // any valid (non-float) numeric literal. + MOZ_ASSERT(CallArgListLength(pn) == 1); + pn = CallArgList(pn); + double d = ExtractNumericNonFloatValue(pn); + return NumLit(NumLit::Float, DoubleValue(d)); + } + + double d = ExtractNumericNonFloatValue(pn, &pn); + + // The asm.js spec syntactically distinguishes any literal containing a + // decimal point or the literal -0 as having double type. + if (NumberNodeHasFrac(pn) || IsNegativeZero(d)) { + return NumLit(NumLit::Double, DoubleValue(d)); + } + + // The syntactic checks above rule out these double values. + MOZ_ASSERT(!IsNegativeZero(d)); + MOZ_ASSERT(!std::isnan(d)); + + // Although doubles can only *precisely* represent 53-bit integers, they + // can *imprecisely* represent integers much bigger than an int64_t. + // Furthermore, d may be inf or -inf. In both cases, casting to an int64_t + // is undefined, so test against the integer bounds using doubles. + if (d < double(INT32_MIN) || d > double(UINT32_MAX)) { + return NumLit(NumLit::OutOfRangeInt, UndefinedValue()); + } + + // With the above syntactic and range limitations, d is definitely an + // integer in the range [INT32_MIN, UINT32_MAX] range. + int64_t i64 = int64_t(d); + if (i64 >= 0) { + if (i64 <= INT32_MAX) { + return NumLit(NumLit::Fixnum, Int32Value(i64)); + } + MOZ_ASSERT(i64 <= UINT32_MAX); + return NumLit(NumLit::BigUnsigned, Int32Value(uint32_t(i64))); + } + MOZ_ASSERT(i64 >= INT32_MIN); + return NumLit(NumLit::NegativeInt, Int32Value(i64)); +} + +static inline bool IsLiteralInt(const NumLit& lit, uint32_t* u32) { + switch (lit.which()) { + case NumLit::Fixnum: + case NumLit::BigUnsigned: + case NumLit::NegativeInt: + *u32 = lit.toUint32(); + return true; + case NumLit::Double: + case NumLit::Float: + case NumLit::OutOfRangeInt: + return false; + } + MOZ_CRASH("Bad literal type"); +} + +static inline bool IsLiteralInt(ModuleValidatorShared& m, ParseNode* pn, + uint32_t* u32) { + return IsNumericLiteral(m, pn) && + IsLiteralInt(ExtractNumericLiteral(m, pn), u32); +} + +/*****************************************************************************/ + +namespace { + +using LabelVector = Vector; + +class MOZ_STACK_CLASS FunctionValidatorShared { + public: + struct Local { + Type type; + unsigned slot; + Local(Type t, unsigned slot) : type(t), slot(slot) { + MOZ_ASSERT(type.isCanonicalValType()); + } + }; + + protected: + using LocalMap = + HashMap; + using LabelMap = + HashMap; + + // This is also a ModuleValidator& after the appropriate static_cast<>. + ModuleValidatorShared& m_; + + FunctionNode* fn_; + Bytes bytes_; + Encoder encoder_; + Uint32Vector callSiteLineNums_; + LocalMap locals_; + + // Labels + LabelMap breakLabels_; + LabelMap continueLabels_; + Uint32Vector breakableStack_; + Uint32Vector continuableStack_; + uint32_t blockDepth_; + + bool hasAlreadyReturned_; + Maybe ret_; + + private: + FunctionValidatorShared(ModuleValidatorShared& m, FunctionNode* fn, + FrontendContext* fc) + : m_(m), + fn_(fn), + encoder_(bytes_), + locals_(fc), + breakLabels_(fc), + continueLabels_(fc), + blockDepth_(0), + hasAlreadyReturned_(false) {} + + protected: + template + FunctionValidatorShared(ModuleValidator& m, FunctionNode* fn, + FrontendContext* fc) + : FunctionValidatorShared(static_cast(m), fn, + fc) {} + + public: + ModuleValidatorShared& m() const { return m_; } + + FrontendContext* fc() const { return m_.fc(); } + FunctionNode* fn() const { return fn_; } + + void define(ModuleValidatorShared::Func* func, unsigned line) { + MOZ_ASSERT(!blockDepth_); + MOZ_ASSERT(breakableStack_.empty()); + MOZ_ASSERT(continuableStack_.empty()); + MOZ_ASSERT(breakLabels_.empty()); + MOZ_ASSERT(continueLabels_.empty()); + func->define(fn_, line, std::move(bytes_), std::move(callSiteLineNums_)); + } + + bool fail(ParseNode* pn, const char* str) { return m_.fail(pn, str); } + + bool failf(ParseNode* pn, const char* fmt, ...) MOZ_FORMAT_PRINTF(3, 4) { + va_list ap; + va_start(ap, fmt); + m_.failfVAOffset(pn->pn_pos.begin, fmt, ap); + va_end(ap); + return false; + } + + bool failName(ParseNode* pn, const char* fmt, TaggedParserAtomIndex name) { + return m_.failName(pn, fmt, name); + } + + /***************************************************** Local scope setup */ + + bool addLocal(ParseNode* pn, TaggedParserAtomIndex name, Type type) { + LocalMap::AddPtr p = locals_.lookupForAdd(name); + if (p) { + return failName(pn, "duplicate local name '%s' not allowed", name); + } + return locals_.add(p, name, Local(type, locals_.count())); + } + + /****************************** For consistency of returns in a function */ + + bool hasAlreadyReturned() const { return hasAlreadyReturned_; } + + Maybe returnedType() const { return ret_; } + + void setReturnedType(const Maybe& ret) { + MOZ_ASSERT(!hasAlreadyReturned_); + ret_ = ret; + hasAlreadyReturned_ = true; + } + + /**************************************************************** Labels */ + private: + bool writeBr(uint32_t absolute, Op op = Op::Br) { + MOZ_ASSERT(op == Op::Br || op == Op::BrIf); + MOZ_ASSERT(absolute < blockDepth_); + return encoder().writeOp(op) && + encoder().writeVarU32(blockDepth_ - 1 - absolute); + } + void removeLabel(TaggedParserAtomIndex label, LabelMap* map) { + LabelMap::Ptr p = map->lookup(label); + MOZ_ASSERT(p); + map->remove(p); + } + + public: + bool pushBreakableBlock() { + return encoder().writeOp(Op::Block) && + encoder().writeFixedU8(uint8_t(TypeCode::BlockVoid)) && + breakableStack_.append(blockDepth_++); + } + bool popBreakableBlock() { + MOZ_ALWAYS_TRUE(breakableStack_.popCopy() == --blockDepth_); + return encoder().writeOp(Op::End); + } + + bool pushUnbreakableBlock(const LabelVector* labels = nullptr) { + if (labels) { + for (TaggedParserAtomIndex label : *labels) { + if (!breakLabels_.putNew(label, blockDepth_)) { + return false; + } + } + } + blockDepth_++; + return encoder().writeOp(Op::Block) && + encoder().writeFixedU8(uint8_t(TypeCode::BlockVoid)); + } + bool popUnbreakableBlock(const LabelVector* labels = nullptr) { + if (labels) { + for (TaggedParserAtomIndex label : *labels) { + removeLabel(label, &breakLabels_); + } + } + --blockDepth_; + return encoder().writeOp(Op::End); + } + + bool pushContinuableBlock() { + return encoder().writeOp(Op::Block) && + encoder().writeFixedU8(uint8_t(TypeCode::BlockVoid)) && + continuableStack_.append(blockDepth_++); + } + bool popContinuableBlock() { + MOZ_ALWAYS_TRUE(continuableStack_.popCopy() == --blockDepth_); + return encoder().writeOp(Op::End); + } + + bool pushLoop() { + return encoder().writeOp(Op::Block) && + encoder().writeFixedU8(uint8_t(TypeCode::BlockVoid)) && + encoder().writeOp(Op::Loop) && + encoder().writeFixedU8(uint8_t(TypeCode::BlockVoid)) && + breakableStack_.append(blockDepth_++) && + continuableStack_.append(blockDepth_++); + } + bool popLoop() { + MOZ_ALWAYS_TRUE(continuableStack_.popCopy() == --blockDepth_); + MOZ_ALWAYS_TRUE(breakableStack_.popCopy() == --blockDepth_); + return encoder().writeOp(Op::End) && encoder().writeOp(Op::End); + } + + bool pushIf(size_t* typeAt) { + ++blockDepth_; + return encoder().writeOp(Op::If) && encoder().writePatchableFixedU7(typeAt); + } + bool switchToElse() { + MOZ_ASSERT(blockDepth_ > 0); + return encoder().writeOp(Op::Else); + } + void setIfType(size_t typeAt, TypeCode type) { + encoder().patchFixedU7(typeAt, uint8_t(type)); + } + bool popIf() { + MOZ_ASSERT(blockDepth_ > 0); + --blockDepth_; + return encoder().writeOp(Op::End); + } + bool popIf(size_t typeAt, TypeCode type) { + MOZ_ASSERT(blockDepth_ > 0); + --blockDepth_; + if (!encoder().writeOp(Op::End)) { + return false; + } + + setIfType(typeAt, type); + return true; + } + + bool writeBreakIf() { return writeBr(breakableStack_.back(), Op::BrIf); } + bool writeContinueIf() { return writeBr(continuableStack_.back(), Op::BrIf); } + bool writeUnlabeledBreakOrContinue(bool isBreak) { + return writeBr(isBreak ? breakableStack_.back() : continuableStack_.back()); + } + bool writeContinue() { return writeBr(continuableStack_.back()); } + + bool addLabels(const LabelVector& labels, uint32_t relativeBreakDepth, + uint32_t relativeContinueDepth) { + for (TaggedParserAtomIndex label : labels) { + if (!breakLabels_.putNew(label, blockDepth_ + relativeBreakDepth)) { + return false; + } + if (!continueLabels_.putNew(label, blockDepth_ + relativeContinueDepth)) { + return false; + } + } + return true; + } + void removeLabels(const LabelVector& labels) { + for (TaggedParserAtomIndex label : labels) { + removeLabel(label, &breakLabels_); + removeLabel(label, &continueLabels_); + } + } + bool writeLabeledBreakOrContinue(TaggedParserAtomIndex label, bool isBreak) { + LabelMap& map = isBreak ? breakLabels_ : continueLabels_; + if (LabelMap::Ptr p = map.lookup(label)) { + return writeBr(p->value()); + } + MOZ_CRASH("nonexistent label"); + } + + /*************************************************** Read-only interface */ + + const Local* lookupLocal(TaggedParserAtomIndex name) const { + if (auto p = locals_.lookup(name)) { + return &p->value(); + } + return nullptr; + } + + const ModuleValidatorShared::Global* lookupGlobal( + TaggedParserAtomIndex name) const { + if (locals_.has(name)) { + return nullptr; + } + return m_.lookupGlobal(name); + } + + size_t numLocals() const { return locals_.count(); } + + /**************************************************** Encoding interface */ + + Encoder& encoder() { return encoder_; } + + [[nodiscard]] bool writeInt32Lit(int32_t i32) { + return encoder().writeOp(Op::I32Const) && encoder().writeVarS32(i32); + } + [[nodiscard]] bool writeConstExpr(const NumLit& lit) { + switch (lit.which()) { + case NumLit::Fixnum: + case NumLit::NegativeInt: + case NumLit::BigUnsigned: + return writeInt32Lit(lit.toInt32()); + case NumLit::Float: + return encoder().writeOp(Op::F32Const) && + encoder().writeFixedF32(lit.toFloat()); + case NumLit::Double: + return encoder().writeOp(Op::F64Const) && + encoder().writeFixedF64(lit.toDouble()); + case NumLit::OutOfRangeInt: + break; + } + MOZ_CRASH("unexpected literal type"); + } +}; + +// Encapsulates the building of an asm bytecode function from an asm.js function +// source code, packing the asm.js code into the asm bytecode form that can +// be decoded and compiled with a FunctionCompiler. +template +class MOZ_STACK_CLASS FunctionValidator : public FunctionValidatorShared { + public: + FunctionValidator(ModuleValidator& m, FunctionNode* fn) + : FunctionValidatorShared(m, fn, m.fc()) {} + + public: + ModuleValidator& m() const { + return static_cast&>(FunctionValidatorShared::m()); + } + + [[nodiscard]] bool writeCall(ParseNode* pn, Op op) { + MOZ_ASSERT(op == Op::Call); + if (!encoder().writeOp(op)) { + return false; + } + + return appendCallSiteLineNumber(pn); + } + [[nodiscard]] bool writeCall(ParseNode* pn, MozOp op) { + MOZ_ASSERT(op == MozOp::OldCallDirect || op == MozOp::OldCallIndirect); + if (!encoder().writeOp(op)) { + return false; + } + + return appendCallSiteLineNumber(pn); + } + [[nodiscard]] bool prepareCall(ParseNode* pn) { + return appendCallSiteLineNumber(pn); + } + + private: + [[nodiscard]] bool appendCallSiteLineNumber(ParseNode* node) { + const TokenStreamAnyChars& anyChars = m().tokenStream().anyCharsAccess(); + auto lineToken = anyChars.lineToken(node->pn_pos.begin); + uint32_t lineNumber = anyChars.lineNumber(lineToken); + if (lineNumber > CallSiteDesc::MAX_LINE_OR_BYTECODE_VALUE) { + return fail(node, "line number exceeding implementation limits"); + } + return callSiteLineNums_.append(lineNumber); + } +}; + +} /* anonymous namespace */ + +/*****************************************************************************/ +// asm.js type-checking and code-generation algorithm + +static bool CheckIdentifier(ModuleValidatorShared& m, ParseNode* usepn, + TaggedParserAtomIndex name) { + if (name == TaggedParserAtomIndex::WellKnown::arguments() || + name == TaggedParserAtomIndex::WellKnown::eval()) { + return m.failName(usepn, "'%s' is not an allowed identifier", name); + } + return true; +} + +static bool CheckModuleLevelName(ModuleValidatorShared& m, ParseNode* usepn, + TaggedParserAtomIndex name) { + if (!CheckIdentifier(m, usepn, name)) { + return false; + } + + if (name == m.moduleFunctionName() || name == m.globalArgumentName() || + name == m.importArgumentName() || name == m.bufferArgumentName() || + m.lookupGlobal(name)) { + return m.failName(usepn, "duplicate name '%s' not allowed", name); + } + + return true; +} + +static bool CheckFunctionHead(ModuleValidatorShared& m, FunctionNode* funNode) { + FunctionBox* funbox = funNode->funbox(); + MOZ_ASSERT(!funbox->hasExprBody()); + + if (funbox->hasRest()) { + return m.fail(funNode, "rest args not allowed"); + } + if (funbox->hasDestructuringArgs) { + return m.fail(funNode, "destructuring args not allowed"); + } + return true; +} + +static bool CheckArgument(ModuleValidatorShared& m, ParseNode* arg, + TaggedParserAtomIndex* name) { + *name = TaggedParserAtomIndex::null(); + + if (!arg->isKind(ParseNodeKind::Name)) { + return m.fail(arg, "argument is not a plain name"); + } + + TaggedParserAtomIndex argName = arg->as().name(); + if (!CheckIdentifier(m, arg, argName)) { + return false; + } + + *name = argName; + return true; +} + +static bool CheckModuleArgument(ModuleValidatorShared& m, ParseNode* arg, + TaggedParserAtomIndex* name) { + if (!CheckArgument(m, arg, name)) { + return false; + } + + if (!CheckModuleLevelName(m, arg, *name)) { + return false; + } + + return true; +} + +static bool CheckModuleArguments(ModuleValidatorShared& m, + FunctionNode* funNode) { + unsigned numFormals; + ParseNode* arg1 = FunctionFormalParametersList(funNode, &numFormals); + ParseNode* arg2 = arg1 ? NextNode(arg1) : nullptr; + ParseNode* arg3 = arg2 ? NextNode(arg2) : nullptr; + + if (numFormals > 3) { + return m.fail(funNode, "asm.js modules takes at most 3 argument"); + } + + TaggedParserAtomIndex arg1Name; + if (arg1 && !CheckModuleArgument(m, arg1, &arg1Name)) { + return false; + } + if (!m.initGlobalArgumentName(arg1Name)) { + return false; + } + + TaggedParserAtomIndex arg2Name; + if (arg2 && !CheckModuleArgument(m, arg2, &arg2Name)) { + return false; + } + if (!m.initImportArgumentName(arg2Name)) { + return false; + } + + TaggedParserAtomIndex arg3Name; + if (arg3 && !CheckModuleArgument(m, arg3, &arg3Name)) { + return false; + } + if (!m.initBufferArgumentName(arg3Name)) { + return false; + } + + return true; +} + +static bool CheckPrecedingStatements(ModuleValidatorShared& m, + ParseNode* stmtList) { + MOZ_ASSERT(stmtList->isKind(ParseNodeKind::StatementList)); + + ParseNode* stmt = ListHead(stmtList); + for (unsigned i = 0, n = ListLength(stmtList); i < n; i++) { + if (!IsIgnoredDirective(stmt)) { + return m.fail(stmt, "invalid asm.js statement"); + } + } + + return true; +} + +static bool CheckGlobalVariableInitConstant(ModuleValidatorShared& m, + TaggedParserAtomIndex varName, + ParseNode* initNode, bool isConst) { + NumLit lit = ExtractNumericLiteral(m, initNode); + if (!lit.valid()) { + return m.fail(initNode, + "global initializer is out of representable integer range"); + } + + Type canonicalType = Type::canonicalize(Type::lit(lit)); + if (!canonicalType.isGlobalVarType()) { + return m.fail(initNode, "global variable type not allowed"); + } + + return m.addGlobalVarInit(varName, lit, canonicalType, isConst); +} + +static bool CheckTypeAnnotation(ModuleValidatorShared& m, + ParseNode* coercionNode, Type* coerceTo, + ParseNode** coercedExpr = nullptr) { + switch (coercionNode->getKind()) { + case ParseNodeKind::BitOrExpr: { + ParseNode* rhs = BitwiseRight(coercionNode); + uint32_t i; + if (!IsLiteralInt(m, rhs, &i) || i != 0) { + return m.fail(rhs, "must use |0 for argument/return coercion"); + } + *coerceTo = Type::Int; + if (coercedExpr) { + *coercedExpr = BitwiseLeft(coercionNode); + } + return true; + } + case ParseNodeKind::PosExpr: { + *coerceTo = Type::Double; + if (coercedExpr) { + *coercedExpr = UnaryKid(coercionNode); + } + return true; + } + case ParseNodeKind::CallExpr: { + if (IsCoercionCall(m, coercionNode, coerceTo, coercedExpr)) { + return true; + } + break; + } + default:; + } + + return m.fail(coercionNode, "must be of the form +x, x|0 or fround(x)"); +} + +static bool CheckGlobalVariableInitImport(ModuleValidatorShared& m, + TaggedParserAtomIndex varName, + ParseNode* initNode, bool isConst) { + Type coerceTo; + ParseNode* coercedExpr; + if (!CheckTypeAnnotation(m, initNode, &coerceTo, &coercedExpr)) { + return false; + } + + if (!coercedExpr->isKind(ParseNodeKind::DotExpr)) { + return m.failName(coercedExpr, "invalid import expression for global '%s'", + varName); + } + + if (!coerceTo.isGlobalVarType()) { + return m.fail(initNode, "global variable type not allowed"); + } + + ParseNode* base = DotBase(coercedExpr); + TaggedParserAtomIndex field = DotMember(coercedExpr); + + TaggedParserAtomIndex importName = m.importArgumentName(); + if (!importName) { + return m.fail(coercedExpr, + "cannot import without an asm.js foreign parameter"); + } + if (!IsUseOfName(base, importName)) { + return m.failName(coercedExpr, "base of import expression must be '%s'", + importName); + } + + return m.addGlobalVarImport(varName, field, coerceTo, isConst); +} + +static bool IsArrayViewCtorName(ModuleValidatorShared& m, + TaggedParserAtomIndex name, + Scalar::Type* type) { + if (name == TaggedParserAtomIndex::WellKnown::Int8Array()) { + *type = Scalar::Int8; + } else if (name == TaggedParserAtomIndex::WellKnown::Uint8Array()) { + *type = Scalar::Uint8; + } else if (name == TaggedParserAtomIndex::WellKnown::Int16Array()) { + *type = Scalar::Int16; + } else if (name == TaggedParserAtomIndex::WellKnown::Uint16Array()) { + *type = Scalar::Uint16; + } else if (name == TaggedParserAtomIndex::WellKnown::Int32Array()) { + *type = Scalar::Int32; + } else if (name == TaggedParserAtomIndex::WellKnown::Uint32Array()) { + *type = Scalar::Uint32; + } else if (name == TaggedParserAtomIndex::WellKnown::Float32Array()) { + *type = Scalar::Float32; + } else if (name == TaggedParserAtomIndex::WellKnown::Float64Array()) { + *type = Scalar::Float64; + } else { + return false; + } + return true; +} + +static bool CheckNewArrayViewArgs(ModuleValidatorShared& m, ParseNode* newExpr, + TaggedParserAtomIndex bufferName) { + ParseNode* ctorExpr = BinaryLeft(newExpr); + ParseNode* ctorArgs = BinaryRight(newExpr); + ParseNode* bufArg = ListHead(ctorArgs); + if (!bufArg || NextNode(bufArg) != nullptr) { + return m.fail(ctorExpr, + "array view constructor takes exactly one argument"); + } + + if (!IsUseOfName(bufArg, bufferName)) { + return m.failName(bufArg, "argument to array view constructor must be '%s'", + bufferName); + } + + return true; +} + +static bool CheckNewArrayView(ModuleValidatorShared& m, + TaggedParserAtomIndex varName, + ParseNode* newExpr) { + TaggedParserAtomIndex globalName = m.globalArgumentName(); + if (!globalName) { + return m.fail( + newExpr, "cannot create array view without an asm.js global parameter"); + } + + TaggedParserAtomIndex bufferName = m.bufferArgumentName(); + if (!bufferName) { + return m.fail(newExpr, + "cannot create array view without an asm.js heap parameter"); + } + + ParseNode* ctorExpr = BinaryLeft(newExpr); + + TaggedParserAtomIndex field; + Scalar::Type type; + if (ctorExpr->isKind(ParseNodeKind::DotExpr)) { + ParseNode* base = DotBase(ctorExpr); + + if (!IsUseOfName(base, globalName)) { + return m.failName(base, "expecting '%s.*Array", globalName); + } + + field = DotMember(ctorExpr); + if (!IsArrayViewCtorName(m, field, &type)) { + return m.fail(ctorExpr, "could not match typed array name"); + } + } else { + if (!ctorExpr->isKind(ParseNodeKind::Name)) { + return m.fail(ctorExpr, + "expecting name of imported array view constructor"); + } + + TaggedParserAtomIndex globalName = ctorExpr->as().name(); + const ModuleValidatorShared::Global* global = m.lookupGlobal(globalName); + if (!global) { + return m.failName(ctorExpr, "%s not found in module global scope", + globalName); + } + + if (global->which() != ModuleValidatorShared::Global::ArrayViewCtor) { + return m.failName(ctorExpr, + "%s must be an imported array view constructor", + globalName); + } + + type = global->viewType(); + } + + if (!CheckNewArrayViewArgs(m, newExpr, bufferName)) { + return false; + } + + return m.addArrayView(varName, type, field); +} + +static bool CheckGlobalMathImport(ModuleValidatorShared& m, ParseNode* initNode, + TaggedParserAtomIndex varName, + TaggedParserAtomIndex field) { + // Math builtin, with the form glob.Math.[[builtin]] + ModuleValidatorShared::MathBuiltin mathBuiltin; + if (!m.lookupStandardLibraryMathName(field, &mathBuiltin)) { + return m.failName(initNode, "'%s' is not a standard Math builtin", field); + } + + switch (mathBuiltin.kind) { + case ModuleValidatorShared::MathBuiltin::Function: + return m.addMathBuiltinFunction(varName, mathBuiltin.u.func, field); + case ModuleValidatorShared::MathBuiltin::Constant: + return m.addMathBuiltinConstant(varName, mathBuiltin.u.cst, field); + default: + break; + } + MOZ_CRASH("unexpected or uninitialized math builtin type"); +} + +static bool CheckGlobalDotImport(ModuleValidatorShared& m, + TaggedParserAtomIndex varName, + ParseNode* initNode) { + ParseNode* base = DotBase(initNode); + TaggedParserAtomIndex field = DotMember(initNode); + + if (base->isKind(ParseNodeKind::DotExpr)) { + ParseNode* global = DotBase(base); + TaggedParserAtomIndex math = DotMember(base); + + TaggedParserAtomIndex globalName = m.globalArgumentName(); + if (!globalName) { + return m.fail( + base, "import statement requires the module have a stdlib parameter"); + } + + if (!IsUseOfName(global, globalName)) { + if (global->isKind(ParseNodeKind::DotExpr)) { + return m.failName(base, + "imports can have at most two dot accesses " + "(e.g. %s.Math.sin)", + globalName); + } + return m.failName(base, "expecting %s.*", globalName); + } + + if (math == TaggedParserAtomIndex::WellKnown::Math()) { + return CheckGlobalMathImport(m, initNode, varName, field); + } + return m.failName(base, "expecting %s.Math", globalName); + } + + if (!base->isKind(ParseNodeKind::Name)) { + return m.fail(base, "expected name of variable or parameter"); + } + + auto baseName = base->as().name(); + if (baseName == m.globalArgumentName()) { + if (field == TaggedParserAtomIndex::WellKnown::NaN()) { + return m.addGlobalConstant(varName, GenericNaN(), field); + } + if (field == TaggedParserAtomIndex::WellKnown::Infinity()) { + return m.addGlobalConstant(varName, PositiveInfinity(), field); + } + + Scalar::Type type; + if (IsArrayViewCtorName(m, field, &type)) { + return m.addArrayViewCtor(varName, type, field); + } + + return m.failName( + initNode, "'%s' is not a standard constant or typed array name", field); + } + + if (baseName != m.importArgumentName()) { + return m.fail(base, "expected global or import name"); + } + + return m.addFFI(varName, field); +} + +static bool CheckModuleGlobal(ModuleValidatorShared& m, ParseNode* decl, + bool isConst) { + if (!decl->isKind(ParseNodeKind::AssignExpr)) { + return m.fail(decl, "module import needs initializer"); + } + AssignmentNode* assignNode = &decl->as(); + + ParseNode* var = assignNode->left(); + + if (!var->isKind(ParseNodeKind::Name)) { + return m.fail(var, "import variable is not a plain name"); + } + + TaggedParserAtomIndex varName = var->as().name(); + if (!CheckModuleLevelName(m, var, varName)) { + return false; + } + + ParseNode* initNode = assignNode->right(); + + if (IsNumericLiteral(m, initNode)) { + return CheckGlobalVariableInitConstant(m, varName, initNode, isConst); + } + + if (initNode->isKind(ParseNodeKind::BitOrExpr) || + initNode->isKind(ParseNodeKind::PosExpr) || + initNode->isKind(ParseNodeKind::CallExpr)) { + return CheckGlobalVariableInitImport(m, varName, initNode, isConst); + } + + if (initNode->isKind(ParseNodeKind::NewExpr)) { + return CheckNewArrayView(m, varName, initNode); + } + + if (initNode->isKind(ParseNodeKind::DotExpr)) { + return CheckGlobalDotImport(m, varName, initNode); + } + + return m.fail(initNode, "unsupported import expression"); +} + +template +static bool CheckModuleProcessingDirectives(ModuleValidator& m) { + auto& ts = m.parser().tokenStream; + while (true) { + bool matched; + if (!ts.matchToken(&matched, TokenKind::String, + TokenStreamShared::SlashIsRegExp)) { + return false; + } + if (!matched) { + return true; + } + + if (!IsIgnoredDirectiveName(ts.anyCharsAccess().currentToken().atom())) { + return m.failCurrentOffset("unsupported processing directive"); + } + + TokenKind tt; + if (!ts.getToken(&tt)) { + return false; + } + if (tt != TokenKind::Semi) { + return m.failCurrentOffset("expected semicolon after string literal"); + } + } +} + +template +static bool CheckModuleGlobals(ModuleValidator& m) { + while (true) { + ParseNode* varStmt; + if (!ParseVarOrConstStatement(m.parser(), &varStmt)) { + return false; + } + if (!varStmt) { + break; + } + for (ParseNode* var = VarListHead(varStmt); var; var = NextNode(var)) { + if (!CheckModuleGlobal(m, var, + varStmt->isKind(ParseNodeKind::ConstDecl))) { + return false; + } + } + } + + return true; +} + +static bool ArgFail(FunctionValidatorShared& f, TaggedParserAtomIndex argName, + ParseNode* stmt) { + return f.failName(stmt, + "expecting argument type declaration for '%s' of the " + "form 'arg = arg|0' or 'arg = +arg' or 'arg = fround(arg)'", + argName); +} + +static bool CheckArgumentType(FunctionValidatorShared& f, ParseNode* stmt, + TaggedParserAtomIndex name, Type* type) { + if (!stmt || !IsExpressionStatement(stmt)) { + return ArgFail(f, name, stmt ? stmt : f.fn()); + } + + ParseNode* initNode = ExpressionStatementExpr(stmt); + if (!initNode->isKind(ParseNodeKind::AssignExpr)) { + return ArgFail(f, name, stmt); + } + + ParseNode* argNode = BinaryLeft(initNode); + ParseNode* coercionNode = BinaryRight(initNode); + + if (!IsUseOfName(argNode, name)) { + return ArgFail(f, name, stmt); + } + + ParseNode* coercedExpr; + if (!CheckTypeAnnotation(f.m(), coercionNode, type, &coercedExpr)) { + return false; + } + + if (!type->isArgType()) { + return f.failName(stmt, "invalid type for argument '%s'", name); + } + + if (!IsUseOfName(coercedExpr, name)) { + return ArgFail(f, name, stmt); + } + + return true; +} + +static bool CheckProcessingDirectives(ModuleValidatorShared& m, + ParseNode** stmtIter) { + ParseNode* stmt = *stmtIter; + + while (stmt && IsIgnoredDirective(stmt)) { + stmt = NextNode(stmt); + } + + *stmtIter = stmt; + return true; +} + +static bool CheckArguments(FunctionValidatorShared& f, ParseNode** stmtIter, + ValTypeVector* argTypes) { + ParseNode* stmt = *stmtIter; + + unsigned numFormals; + ParseNode* argpn = FunctionFormalParametersList(f.fn(), &numFormals); + + for (unsigned i = 0; i < numFormals; + i++, argpn = NextNode(argpn), stmt = NextNode(stmt)) { + TaggedParserAtomIndex name; + if (!CheckArgument(f.m(), argpn, &name)) { + return false; + } + + Type type; + if (!CheckArgumentType(f, stmt, name, &type)) { + return false; + } + + if (!argTypes->append(type.canonicalToValType())) { + return false; + } + + if (!f.addLocal(argpn, name, type)) { + return false; + } + } + + *stmtIter = stmt; + return true; +} + +static bool IsLiteralOrConst(FunctionValidatorShared& f, ParseNode* pn, + NumLit* lit) { + if (pn->isKind(ParseNodeKind::Name)) { + const ModuleValidatorShared::Global* global = + f.lookupGlobal(pn->as().name()); + if (!global || + global->which() != ModuleValidatorShared::Global::ConstantLiteral) { + return false; + } + + *lit = global->constLiteralValue(); + return true; + } + + if (!IsNumericLiteral(f.m(), pn)) { + return false; + } + + *lit = ExtractNumericLiteral(f.m(), pn); + return true; +} + +static bool CheckFinalReturn(FunctionValidatorShared& f, + ParseNode* lastNonEmptyStmt) { + if (!f.encoder().writeOp(Op::End)) { + return false; + } + + if (!f.hasAlreadyReturned()) { + f.setReturnedType(Nothing()); + return true; + } + + if (!lastNonEmptyStmt->isKind(ParseNodeKind::ReturnStmt) && + f.returnedType()) { + return f.fail(lastNonEmptyStmt, + "void incompatible with previous return type"); + } + + return true; +} + +static bool CheckVariable(FunctionValidatorShared& f, ParseNode* decl, + ValTypeVector* types, Vector* inits) { + if (!decl->isKind(ParseNodeKind::AssignExpr)) { + return f.failName( + decl, "var '%s' needs explicit type declaration via an initial value", + decl->as().name()); + } + AssignmentNode* assignNode = &decl->as(); + + ParseNode* var = assignNode->left(); + + if (!var->isKind(ParseNodeKind::Name)) { + return f.fail(var, "local variable is not a plain name"); + } + + TaggedParserAtomIndex name = var->as().name(); + + if (!CheckIdentifier(f.m(), var, name)) { + return false; + } + + ParseNode* initNode = assignNode->right(); + + NumLit lit; + if (!IsLiteralOrConst(f, initNode, &lit)) { + return f.failName( + var, "var '%s' initializer must be literal or const literal", name); + } + + if (!lit.valid()) { + return f.failName(var, "var '%s' initializer out of range", name); + } + + Type type = Type::canonicalize(Type::lit(lit)); + + return f.addLocal(var, name, type) && + types->append(type.canonicalToValType()) && inits->append(lit); +} + +static bool CheckVariables(FunctionValidatorShared& f, ParseNode** stmtIter) { + ParseNode* stmt = *stmtIter; + + uint32_t firstVar = f.numLocals(); + + ValTypeVector types; + Vector inits(f.fc()); + + for (; stmt && stmt->isKind(ParseNodeKind::VarStmt); + stmt = NextNonEmptyStatement(stmt)) { + for (ParseNode* var = VarListHead(stmt); var; var = NextNode(var)) { + if (!CheckVariable(f, var, &types, &inits)) { + return false; + } + } + } + + MOZ_ASSERT(f.encoder().empty()); + + if (!EncodeLocalEntries(f.encoder(), types)) { + return false; + } + + for (uint32_t i = 0; i < inits.length(); i++) { + NumLit lit = inits[i]; + if (lit.isZeroBits()) { + continue; + } + if (!f.writeConstExpr(lit)) { + return false; + } + if (!f.encoder().writeOp(Op::LocalSet)) { + return false; + } + if (!f.encoder().writeVarU32(firstVar + i)) { + return false; + } + } + + *stmtIter = stmt; + return true; +} + +template +static bool CheckExpr(FunctionValidator& f, ParseNode* expr, Type* type); + +template +static bool CheckNumericLiteral(FunctionValidator& f, ParseNode* num, + Type* type) { + NumLit lit = ExtractNumericLiteral(f.m(), num); + if (!lit.valid()) { + return f.fail(num, "numeric literal out of representable integer range"); + } + *type = Type::lit(lit); + return f.writeConstExpr(lit); +} + +static bool CheckVarRef(FunctionValidatorShared& f, ParseNode* varRef, + Type* type) { + TaggedParserAtomIndex name = varRef->as().name(); + + if (const FunctionValidatorShared::Local* local = f.lookupLocal(name)) { + if (!f.encoder().writeOp(Op::LocalGet)) { + return false; + } + if (!f.encoder().writeVarU32(local->slot)) { + return false; + } + *type = local->type; + return true; + } + + if (const ModuleValidatorShared::Global* global = f.lookupGlobal(name)) { + switch (global->which()) { + case ModuleValidatorShared::Global::ConstantLiteral: + *type = global->varOrConstType(); + return f.writeConstExpr(global->constLiteralValue()); + case ModuleValidatorShared::Global::ConstantImport: + case ModuleValidatorShared::Global::Variable: { + *type = global->varOrConstType(); + return f.encoder().writeOp(Op::GlobalGet) && + f.encoder().writeVarU32(global->varOrConstIndex()); + } + case ModuleValidatorShared::Global::Function: + case ModuleValidatorShared::Global::FFI: + case ModuleValidatorShared::Global::MathBuiltinFunction: + case ModuleValidatorShared::Global::Table: + case ModuleValidatorShared::Global::ArrayView: + case ModuleValidatorShared::Global::ArrayViewCtor: + break; + } + return f.failName(varRef, + "'%s' may not be accessed by ordinary expressions", name); + } + + return f.failName(varRef, "'%s' not found in local or asm.js module scope", + name); +} + +static inline bool IsLiteralOrConstInt(FunctionValidatorShared& f, + ParseNode* pn, uint32_t* u32) { + NumLit lit; + if (!IsLiteralOrConst(f, pn, &lit)) { + return false; + } + + return IsLiteralInt(lit, u32); +} + +static const int32_t NoMask = -1; + +template +static bool CheckArrayAccess(FunctionValidator& f, ParseNode* viewName, + ParseNode* indexExpr, Scalar::Type* viewType) { + if (!viewName->isKind(ParseNodeKind::Name)) { + return f.fail(viewName, + "base of array access must be a typed array view name"); + } + + const ModuleValidatorShared::Global* global = + f.lookupGlobal(viewName->as().name()); + if (!global || global->which() != ModuleValidatorShared::Global::ArrayView) { + return f.fail(viewName, + "base of array access must be a typed array view name"); + } + + *viewType = global->viewType(); + + uint32_t index; + if (IsLiteralOrConstInt(f, indexExpr, &index)) { + uint64_t byteOffset = uint64_t(index) << TypedArrayShift(*viewType); + uint64_t width = TypedArrayElemSize(*viewType); + if (!f.m().tryConstantAccess(byteOffset, width)) { + return f.fail(indexExpr, "constant index out of range"); + } + + return f.writeInt32Lit(byteOffset); + } + + // Mask off the low bits to account for the clearing effect of a right shift + // followed by the left shift implicit in the array access. E.g., H32[i>>2] + // loses the low two bits. + int32_t mask = ~(TypedArrayElemSize(*viewType) - 1); + + if (indexExpr->isKind(ParseNodeKind::RshExpr)) { + ParseNode* shiftAmountNode = BitwiseRight(indexExpr); + + uint32_t shift; + if (!IsLiteralInt(f.m(), shiftAmountNode, &shift)) { + return f.failf(shiftAmountNode, "shift amount must be constant"); + } + + unsigned requiredShift = TypedArrayShift(*viewType); + if (shift != requiredShift) { + return f.failf(shiftAmountNode, "shift amount must be %u", requiredShift); + } + + ParseNode* pointerNode = BitwiseLeft(indexExpr); + + Type pointerType; + if (!CheckExpr(f, pointerNode, &pointerType)) { + return false; + } + + if (!pointerType.isIntish()) { + return f.failf(pointerNode, "%s is not a subtype of int", + pointerType.toChars()); + } + } else { + // For legacy scalar access compatibility, accept Int8/Uint8 accesses + // with no shift. + if (TypedArrayShift(*viewType) != 0) { + return f.fail( + indexExpr, + "index expression isn't shifted; must be an Int8/Uint8 access"); + } + + MOZ_ASSERT(mask == NoMask); + + ParseNode* pointerNode = indexExpr; + + Type pointerType; + if (!CheckExpr(f, pointerNode, &pointerType)) { + return false; + } + if (!pointerType.isInt()) { + return f.failf(pointerNode, "%s is not a subtype of int", + pointerType.toChars()); + } + } + + // Don't generate the mask op if there is no need for it which could happen + // for a shift of zero. + if (mask != NoMask) { + return f.writeInt32Lit(mask) && f.encoder().writeOp(Op::I32And); + } + + return true; +} + +static bool WriteArrayAccessFlags(FunctionValidatorShared& f, + Scalar::Type viewType) { + // asm.js only has naturally-aligned accesses. + size_t align = TypedArrayElemSize(viewType); + MOZ_ASSERT(IsPowerOfTwo(align)); + if (!f.encoder().writeFixedU8(CeilingLog2(align))) { + return false; + } + + // asm.js doesn't have constant offsets, so just encode a 0. + if (!f.encoder().writeVarU32(0)) { + return false; + } + + return true; +} + +template +static bool CheckLoadArray(FunctionValidator& f, ParseNode* elem, + Type* type) { + Scalar::Type viewType; + + if (!CheckArrayAccess(f, ElemBase(elem), ElemIndex(elem), &viewType)) { + return false; + } + + switch (viewType) { + case Scalar::Int8: + if (!f.encoder().writeOp(Op::I32Load8S)) return false; + break; + case Scalar::Uint8: + if (!f.encoder().writeOp(Op::I32Load8U)) return false; + break; + case Scalar::Int16: + if (!f.encoder().writeOp(Op::I32Load16S)) return false; + break; + case Scalar::Uint16: + if (!f.encoder().writeOp(Op::I32Load16U)) return false; + break; + case Scalar::Uint32: + case Scalar::Int32: + if (!f.encoder().writeOp(Op::I32Load)) return false; + break; + case Scalar::Float32: + if (!f.encoder().writeOp(Op::F32Load)) return false; + break; + case Scalar::Float64: + if (!f.encoder().writeOp(Op::F64Load)) return false; + break; + default: + MOZ_CRASH("unexpected scalar type"); + } + + switch (viewType) { + case Scalar::Int8: + case Scalar::Int16: + case Scalar::Int32: + case Scalar::Uint8: + case Scalar::Uint16: + case Scalar::Uint32: + *type = Type::Intish; + break; + case Scalar::Float32: + *type = Type::MaybeFloat; + break; + case Scalar::Float64: + *type = Type::MaybeDouble; + break; + default: + MOZ_CRASH("Unexpected array type"); + } + + return WriteArrayAccessFlags(f, viewType); +} + +template +static bool CheckStoreArray(FunctionValidator& f, ParseNode* lhs, + ParseNode* rhs, Type* type) { + Scalar::Type viewType; + if (!CheckArrayAccess(f, ElemBase(lhs), ElemIndex(lhs), &viewType)) { + return false; + } + + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + switch (viewType) { + case Scalar::Int8: + case Scalar::Int16: + case Scalar::Int32: + case Scalar::Uint8: + case Scalar::Uint16: + case Scalar::Uint32: + if (!rhsType.isIntish()) { + return f.failf(lhs, "%s is not a subtype of intish", rhsType.toChars()); + } + break; + case Scalar::Float32: + if (!rhsType.isMaybeDouble() && !rhsType.isFloatish()) { + return f.failf(lhs, "%s is not a subtype of double? or floatish", + rhsType.toChars()); + } + break; + case Scalar::Float64: + if (!rhsType.isMaybeFloat() && !rhsType.isMaybeDouble()) { + return f.failf(lhs, "%s is not a subtype of float? or double?", + rhsType.toChars()); + } + break; + default: + MOZ_CRASH("Unexpected view type"); + } + + switch (viewType) { + case Scalar::Int8: + case Scalar::Uint8: + if (!f.encoder().writeOp(MozOp::I32TeeStore8)) { + return false; + } + break; + case Scalar::Int16: + case Scalar::Uint16: + if (!f.encoder().writeOp(MozOp::I32TeeStore16)) { + return false; + } + break; + case Scalar::Int32: + case Scalar::Uint32: + if (!f.encoder().writeOp(MozOp::I32TeeStore)) { + return false; + } + break; + case Scalar::Float32: + if (rhsType.isFloatish()) { + if (!f.encoder().writeOp(MozOp::F32TeeStore)) { + return false; + } + } else { + if (!f.encoder().writeOp(MozOp::F64TeeStoreF32)) { + return false; + } + } + break; + case Scalar::Float64: + if (rhsType.isFloatish()) { + if (!f.encoder().writeOp(MozOp::F32TeeStoreF64)) { + return false; + } + } else { + if (!f.encoder().writeOp(MozOp::F64TeeStore)) { + return false; + } + } + break; + default: + MOZ_CRASH("unexpected scalar type"); + } + + if (!WriteArrayAccessFlags(f, viewType)) { + return false; + } + + *type = rhsType; + return true; +} + +template +static bool CheckAssignName(FunctionValidator& f, ParseNode* lhs, + ParseNode* rhs, Type* type) { + TaggedParserAtomIndex name = lhs->as().name(); + + if (const FunctionValidatorShared::Local* lhsVar = f.lookupLocal(name)) { + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + if (!f.encoder().writeOp(Op::LocalTee)) { + return false; + } + if (!f.encoder().writeVarU32(lhsVar->slot)) { + return false; + } + + if (!(rhsType <= lhsVar->type)) { + return f.failf(lhs, "%s is not a subtype of %s", rhsType.toChars(), + lhsVar->type.toChars()); + } + *type = rhsType; + return true; + } + + if (const ModuleValidatorShared::Global* global = f.lookupGlobal(name)) { + if (global->which() != ModuleValidatorShared::Global::Variable) { + return f.failName(lhs, "'%s' is not a mutable variable", name); + } + + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + Type globType = global->varOrConstType(); + if (!(rhsType <= globType)) { + return f.failf(lhs, "%s is not a subtype of %s", rhsType.toChars(), + globType.toChars()); + } + if (!f.encoder().writeOp(MozOp::TeeGlobal)) { + return false; + } + if (!f.encoder().writeVarU32(global->varOrConstIndex())) { + return false; + } + + *type = rhsType; + return true; + } + + return f.failName(lhs, "'%s' not found in local or asm.js module scope", + name); +} + +template +static bool CheckAssign(FunctionValidator& f, ParseNode* assign, + Type* type) { + MOZ_ASSERT(assign->isKind(ParseNodeKind::AssignExpr)); + + ParseNode* lhs = BinaryLeft(assign); + ParseNode* rhs = BinaryRight(assign); + + if (lhs->getKind() == ParseNodeKind::ElemExpr) { + return CheckStoreArray(f, lhs, rhs, type); + } + + if (lhs->getKind() == ParseNodeKind::Name) { + return CheckAssignName(f, lhs, rhs, type); + } + + return f.fail( + assign, + "left-hand side of assignment must be a variable or array access"); +} + +template +static bool CheckMathIMul(FunctionValidator& f, ParseNode* call, + Type* type) { + if (CallArgListLength(call) != 2) { + return f.fail(call, "Math.imul must be passed 2 arguments"); + } + + ParseNode* lhs = CallArgList(call); + ParseNode* rhs = NextNode(lhs); + + Type lhsType; + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + if (!lhsType.isIntish()) { + return f.failf(lhs, "%s is not a subtype of intish", lhsType.toChars()); + } + if (!rhsType.isIntish()) { + return f.failf(rhs, "%s is not a subtype of intish", rhsType.toChars()); + } + + *type = Type::Signed; + return f.encoder().writeOp(Op::I32Mul); +} + +template +static bool CheckMathClz32(FunctionValidator& f, ParseNode* call, + Type* type) { + if (CallArgListLength(call) != 1) { + return f.fail(call, "Math.clz32 must be passed 1 argument"); + } + + ParseNode* arg = CallArgList(call); + + Type argType; + if (!CheckExpr(f, arg, &argType)) { + return false; + } + + if (!argType.isIntish()) { + return f.failf(arg, "%s is not a subtype of intish", argType.toChars()); + } + + *type = Type::Fixnum; + return f.encoder().writeOp(Op::I32Clz); +} + +template +static bool CheckMathAbs(FunctionValidator& f, ParseNode* call, + Type* type) { + if (CallArgListLength(call) != 1) { + return f.fail(call, "Math.abs must be passed 1 argument"); + } + + ParseNode* arg = CallArgList(call); + + Type argType; + if (!CheckExpr(f, arg, &argType)) { + return false; + } + + if (argType.isSigned()) { + *type = Type::Unsigned; + return f.encoder().writeOp(MozOp::I32Abs); + } + + if (argType.isMaybeDouble()) { + *type = Type::Double; + return f.encoder().writeOp(Op::F64Abs); + } + + if (argType.isMaybeFloat()) { + *type = Type::Floatish; + return f.encoder().writeOp(Op::F32Abs); + } + + return f.failf(call, "%s is not a subtype of signed, float? or double?", + argType.toChars()); +} + +template +static bool CheckMathSqrt(FunctionValidator& f, ParseNode* call, + Type* type) { + if (CallArgListLength(call) != 1) { + return f.fail(call, "Math.sqrt must be passed 1 argument"); + } + + ParseNode* arg = CallArgList(call); + + Type argType; + if (!CheckExpr(f, arg, &argType)) { + return false; + } + + if (argType.isMaybeDouble()) { + *type = Type::Double; + return f.encoder().writeOp(Op::F64Sqrt); + } + + if (argType.isMaybeFloat()) { + *type = Type::Floatish; + return f.encoder().writeOp(Op::F32Sqrt); + } + + return f.failf(call, "%s is neither a subtype of double? nor float?", + argType.toChars()); +} + +template +static bool CheckMathMinMax(FunctionValidator& f, ParseNode* callNode, + bool isMax, Type* type) { + if (CallArgListLength(callNode) < 2) { + return f.fail(callNode, "Math.min/max must be passed at least 2 arguments"); + } + + ParseNode* firstArg = CallArgList(callNode); + Type firstType; + if (!CheckExpr(f, firstArg, &firstType)) { + return false; + } + + Op op = Op::Limit; + MozOp mozOp = MozOp::Limit; + if (firstType.isMaybeDouble()) { + *type = Type::Double; + firstType = Type::MaybeDouble; + op = isMax ? Op::F64Max : Op::F64Min; + } else if (firstType.isMaybeFloat()) { + *type = Type::Float; + firstType = Type::MaybeFloat; + op = isMax ? Op::F32Max : Op::F32Min; + } else if (firstType.isSigned()) { + *type = Type::Signed; + firstType = Type::Signed; + mozOp = isMax ? MozOp::I32Max : MozOp::I32Min; + } else { + return f.failf(firstArg, "%s is not a subtype of double?, float? or signed", + firstType.toChars()); + } + + unsigned numArgs = CallArgListLength(callNode); + ParseNode* nextArg = NextNode(firstArg); + for (unsigned i = 1; i < numArgs; i++, nextArg = NextNode(nextArg)) { + Type nextType; + if (!CheckExpr(f, nextArg, &nextType)) { + return false; + } + if (!(nextType <= firstType)) { + return f.failf(nextArg, "%s is not a subtype of %s", nextType.toChars(), + firstType.toChars()); + } + + if (op != Op::Limit) { + if (!f.encoder().writeOp(op)) { + return false; + } + } else { + if (!f.encoder().writeOp(mozOp)) { + return false; + } + } + } + + return true; +} + +using CheckArgType = bool (*)(FunctionValidatorShared& f, ParseNode* argNode, + Type type); + +template +static bool CheckCallArgs(FunctionValidator& f, ParseNode* callNode, + ValTypeVector* args) { + ParseNode* argNode = CallArgList(callNode); + for (unsigned i = 0; i < CallArgListLength(callNode); + i++, argNode = NextNode(argNode)) { + Type type; + if (!CheckExpr(f, argNode, &type)) { + return false; + } + + if (!checkArg(f, argNode, type)) { + return false; + } + + if (!args->append(Type::canonicalize(type).canonicalToValType())) { + return false; + } + } + return true; +} + +static bool CheckSignatureAgainstExisting(ModuleValidatorShared& m, + ParseNode* usepn, const FuncType& sig, + const FuncType& existing) { + if (!FuncType::strictlyEquals(sig, existing)) { + return m.failf(usepn, "incompatible argument types to function"); + } + return true; +} + +template +static bool CheckFunctionSignature(ModuleValidator& m, ParseNode* usepn, + FuncType&& sig, TaggedParserAtomIndex name, + ModuleValidatorShared::Func** func) { + if (sig.args().length() > MaxParams) { + return m.failf(usepn, "too many parameters"); + } + + ModuleValidatorShared::Func* existing = m.lookupFuncDef(name); + if (!existing) { + if (!CheckModuleLevelName(m, usepn, name)) { + return false; + } + return m.addFuncDef(name, usepn->pn_pos.begin, std::move(sig), func); + } + + const FuncType& existingSig = + m.env().types->type(existing->sigIndex()).funcType(); + + if (!CheckSignatureAgainstExisting(m, usepn, sig, existingSig)) { + return false; + } + + *func = existing; + return true; +} + +static bool CheckIsArgType(FunctionValidatorShared& f, ParseNode* argNode, + Type type) { + if (!type.isArgType()) { + return f.failf(argNode, "%s is not a subtype of int, float, or double", + type.toChars()); + } + return true; +} + +template +static bool CheckInternalCall(FunctionValidator& f, ParseNode* callNode, + TaggedParserAtomIndex calleeName, Type ret, + Type* type) { + MOZ_ASSERT(ret.isCanonical()); + + ValTypeVector args; + if (!CheckCallArgs(f, callNode, &args)) { + return false; + } + + ValTypeVector results; + Maybe retType = ret.canonicalToReturnType(); + if (retType && !results.append(retType.ref())) { + return false; + } + + FuncType sig(std::move(args), std::move(results)); + + ModuleValidatorShared::Func* callee; + if (!CheckFunctionSignature(f.m(), callNode, std::move(sig), calleeName, + &callee)) { + return false; + } + + if (!f.writeCall(callNode, MozOp::OldCallDirect)) { + return false; + } + + if (!f.encoder().writeVarU32(callee->funcDefIndex())) { + return false; + } + + *type = Type::ret(ret); + return true; +} + +template +static bool CheckFuncPtrTableAgainstExisting(ModuleValidator& m, + ParseNode* usepn, + TaggedParserAtomIndex name, + FuncType&& sig, unsigned mask, + uint32_t* tableIndex) { + if (const ModuleValidatorShared::Global* existing = m.lookupGlobal(name)) { + if (existing->which() != ModuleValidatorShared::Global::Table) { + return m.failName(usepn, "'%s' is not a function-pointer table", name); + } + + ModuleValidatorShared::Table& table = m.table(existing->tableIndex()); + if (mask != table.mask()) { + return m.failf(usepn, "mask does not match previous value (%u)", + table.mask()); + } + + if (!CheckSignatureAgainstExisting( + m, usepn, sig, m.env().types->type(table.sigIndex()).funcType())) { + return false; + } + + *tableIndex = existing->tableIndex(); + return true; + } + + if (!CheckModuleLevelName(m, usepn, name)) { + return false; + } + + if (!m.declareFuncPtrTable(std::move(sig), name, usepn->pn_pos.begin, mask, + tableIndex)) { + return false; + } + + return true; +} + +template +static bool CheckFuncPtrCall(FunctionValidator& f, ParseNode* callNode, + Type ret, Type* type) { + MOZ_ASSERT(ret.isCanonical()); + + ParseNode* callee = CallCallee(callNode); + ParseNode* tableNode = ElemBase(callee); + ParseNode* indexExpr = ElemIndex(callee); + + if (!tableNode->isKind(ParseNodeKind::Name)) { + return f.fail(tableNode, "expecting name of function-pointer array"); + } + + TaggedParserAtomIndex name = tableNode->as().name(); + if (const ModuleValidatorShared::Global* existing = f.lookupGlobal(name)) { + if (existing->which() != ModuleValidatorShared::Global::Table) { + return f.failName( + tableNode, "'%s' is not the name of a function-pointer array", name); + } + } + + if (!indexExpr->isKind(ParseNodeKind::BitAndExpr)) { + return f.fail(indexExpr, + "function-pointer table index expression needs & mask"); + } + + ParseNode* indexNode = BitwiseLeft(indexExpr); + ParseNode* maskNode = BitwiseRight(indexExpr); + + uint32_t mask; + if (!IsLiteralInt(f.m(), maskNode, &mask) || mask == UINT32_MAX || + !IsPowerOfTwo(mask + 1)) { + return f.fail(maskNode, + "function-pointer table index mask value must be a power of " + "two minus 1"); + } + + Type indexType; + if (!CheckExpr(f, indexNode, &indexType)) { + return false; + } + + if (!indexType.isIntish()) { + return f.failf(indexNode, "%s is not a subtype of intish", + indexType.toChars()); + } + + ValTypeVector args; + if (!CheckCallArgs(f, callNode, &args)) { + return false; + } + + ValTypeVector results; + Maybe retType = ret.canonicalToReturnType(); + if (retType && !results.append(retType.ref())) { + return false; + } + + FuncType sig(std::move(args), std::move(results)); + + uint32_t tableIndex; + if (!CheckFuncPtrTableAgainstExisting(f.m(), tableNode, name, std::move(sig), + mask, &tableIndex)) { + return false; + } + + if (!f.writeCall(callNode, MozOp::OldCallIndirect)) { + return false; + } + + // Call signature + if (!f.encoder().writeVarU32(f.m().table(tableIndex).sigIndex())) { + return false; + } + + *type = Type::ret(ret); + return true; +} + +static bool CheckIsExternType(FunctionValidatorShared& f, ParseNode* argNode, + Type type) { + if (!type.isExtern()) { + return f.failf(argNode, "%s is not a subtype of extern", type.toChars()); + } + return true; +} + +template +static bool CheckFFICall(FunctionValidator& f, ParseNode* callNode, + unsigned ffiIndex, Type ret, Type* type) { + MOZ_ASSERT(ret.isCanonical()); + + TaggedParserAtomIndex calleeName = + CallCallee(callNode)->as().name(); + + if (ret.isFloat()) { + return f.fail(callNode, "FFI calls can't return float"); + } + + ValTypeVector args; + if (!CheckCallArgs(f, callNode, &args)) { + return false; + } + + ValTypeVector results; + Maybe retType = ret.canonicalToReturnType(); + if (retType && !results.append(retType.ref())) { + return false; + } + + FuncType sig(std::move(args), std::move(results)); + + uint32_t importIndex; + if (!f.m().declareImport(calleeName, std::move(sig), ffiIndex, + &importIndex)) { + return false; + } + + if (!f.writeCall(callNode, Op::Call)) { + return false; + } + + if (!f.encoder().writeVarU32(importIndex)) { + return false; + } + + *type = Type::ret(ret); + return true; +} + +static bool CheckFloatCoercionArg(FunctionValidatorShared& f, + ParseNode* inputNode, Type inputType) { + if (inputType.isMaybeDouble()) { + return f.encoder().writeOp(Op::F32DemoteF64); + } + if (inputType.isSigned()) { + return f.encoder().writeOp(Op::F32ConvertI32S); + } + if (inputType.isUnsigned()) { + return f.encoder().writeOp(Op::F32ConvertI32U); + } + if (inputType.isFloatish()) { + return true; + } + + return f.failf(inputNode, + "%s is not a subtype of signed, unsigned, double? or floatish", + inputType.toChars()); +} + +template +static bool CheckCoercedCall(FunctionValidator& f, ParseNode* call, + Type ret, Type* type); + +template +static bool CheckCoercionArg(FunctionValidator& f, ParseNode* arg, + Type expected, Type* type) { + MOZ_ASSERT(expected.isCanonicalValType()); + + if (arg->isKind(ParseNodeKind::CallExpr)) { + return CheckCoercedCall(f, arg, expected, type); + } + + Type argType; + if (!CheckExpr(f, arg, &argType)) { + return false; + } + + if (expected.isFloat()) { + if (!CheckFloatCoercionArg(f, arg, argType)) { + return false; + } + } else { + MOZ_CRASH("not call coercions"); + } + + *type = Type::ret(expected); + return true; +} + +template +static bool CheckMathFRound(FunctionValidator& f, ParseNode* callNode, + Type* type) { + if (CallArgListLength(callNode) != 1) { + return f.fail(callNode, "Math.fround must be passed 1 argument"); + } + + ParseNode* argNode = CallArgList(callNode); + Type argType; + if (!CheckCoercionArg(f, argNode, Type::Float, &argType)) { + return false; + } + + MOZ_ASSERT(argType == Type::Float); + *type = Type::Float; + return true; +} + +template +static bool CheckMathBuiltinCall(FunctionValidator& f, + ParseNode* callNode, + AsmJSMathBuiltinFunction func, Type* type) { + unsigned arity = 0; + Op f32 = Op::Limit; + Op f64 = Op::Limit; + MozOp mozf64 = MozOp::Limit; + switch (func) { + case AsmJSMathBuiltin_imul: + return CheckMathIMul(f, callNode, type); + case AsmJSMathBuiltin_clz32: + return CheckMathClz32(f, callNode, type); + case AsmJSMathBuiltin_abs: + return CheckMathAbs(f, callNode, type); + case AsmJSMathBuiltin_sqrt: + return CheckMathSqrt(f, callNode, type); + case AsmJSMathBuiltin_fround: + return CheckMathFRound(f, callNode, type); + case AsmJSMathBuiltin_min: + return CheckMathMinMax(f, callNode, /* isMax = */ false, type); + case AsmJSMathBuiltin_max: + return CheckMathMinMax(f, callNode, /* isMax = */ true, type); + case AsmJSMathBuiltin_ceil: + arity = 1; + f64 = Op::F64Ceil; + f32 = Op::F32Ceil; + break; + case AsmJSMathBuiltin_floor: + arity = 1; + f64 = Op::F64Floor; + f32 = Op::F32Floor; + break; + case AsmJSMathBuiltin_sin: + arity = 1; + if (!f.m().shouldResistFingerprinting()) { + mozf64 = MozOp::F64SinNative; + } else { + mozf64 = MozOp::F64SinFdlibm; + } + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_cos: + arity = 1; + if (!f.m().shouldResistFingerprinting()) { + mozf64 = MozOp::F64CosNative; + } else { + mozf64 = MozOp::F64CosFdlibm; + } + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_tan: + arity = 1; + if (!f.m().shouldResistFingerprinting()) { + mozf64 = MozOp::F64TanNative; + } else { + mozf64 = MozOp::F64TanFdlibm; + } + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_asin: + arity = 1; + mozf64 = MozOp::F64Asin; + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_acos: + arity = 1; + mozf64 = MozOp::F64Acos; + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_atan: + arity = 1; + mozf64 = MozOp::F64Atan; + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_exp: + arity = 1; + mozf64 = MozOp::F64Exp; + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_log: + arity = 1; + mozf64 = MozOp::F64Log; + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_pow: + arity = 2; + mozf64 = MozOp::F64Pow; + f32 = Op::Unreachable; + break; + case AsmJSMathBuiltin_atan2: + arity = 2; + mozf64 = MozOp::F64Atan2; + f32 = Op::Unreachable; + break; + default: + MOZ_CRASH("unexpected mathBuiltin function"); + } + + unsigned actualArity = CallArgListLength(callNode); + if (actualArity != arity) { + return f.failf(callNode, "call passed %u arguments, expected %u", + actualArity, arity); + } + + if (!f.prepareCall(callNode)) { + return false; + } + + Type firstType; + ParseNode* argNode = CallArgList(callNode); + if (!CheckExpr(f, argNode, &firstType)) { + return false; + } + + if (!firstType.isMaybeFloat() && !firstType.isMaybeDouble()) { + return f.fail( + argNode, + "arguments to math call should be a subtype of double? or float?"); + } + + bool opIsDouble = firstType.isMaybeDouble(); + if (!opIsDouble && f32 == Op::Unreachable) { + return f.fail(callNode, "math builtin cannot be used as float"); + } + + if (arity == 2) { + Type secondType; + argNode = NextNode(argNode); + if (!CheckExpr(f, argNode, &secondType)) { + return false; + } + + if (firstType.isMaybeDouble() && !secondType.isMaybeDouble()) { + return f.fail( + argNode, + "both arguments to math builtin call should be the same type"); + } + if (firstType.isMaybeFloat() && !secondType.isMaybeFloat()) { + return f.fail( + argNode, + "both arguments to math builtin call should be the same type"); + } + } + + if (opIsDouble) { + if (f64 != Op::Limit) { + if (!f.encoder().writeOp(f64)) { + return false; + } + } else { + if (!f.encoder().writeOp(mozf64)) { + return false; + } + } + } else { + if (!f.encoder().writeOp(f32)) { + return false; + } + } + + *type = opIsDouble ? Type::Double : Type::Floatish; + return true; +} + +template +static bool CheckUncoercedCall(FunctionValidator& f, ParseNode* expr, + Type* type) { + MOZ_ASSERT(expr->isKind(ParseNodeKind::CallExpr)); + + const ModuleValidatorShared::Global* global; + if (IsCallToGlobal(f.m(), expr, &global) && global->isMathFunction()) { + return CheckMathBuiltinCall(f, expr, global->mathBuiltinFunction(), type); + } + + return f.fail( + expr, + "all function calls must be calls to standard lib math functions," + " ignored (via f(); or comma-expression), coerced to signed (via f()|0)," + " coerced to float (via fround(f())), or coerced to double (via +f())"); +} + +static bool CoerceResult(FunctionValidatorShared& f, ParseNode* expr, + Type expected, Type actual, Type* type) { + MOZ_ASSERT(expected.isCanonical()); + + // At this point, the bytecode resembles this: + // | the thing we wanted to coerce | current position |> + switch (expected.which()) { + case Type::Void: + if (!actual.isVoid()) { + if (!f.encoder().writeOp(Op::Drop)) { + return false; + } + } + break; + case Type::Int: + if (!actual.isIntish()) { + return f.failf(expr, "%s is not a subtype of intish", actual.toChars()); + } + break; + case Type::Float: + if (!CheckFloatCoercionArg(f, expr, actual)) { + return false; + } + break; + case Type::Double: + if (actual.isMaybeDouble()) { + // No conversion necessary. + } else if (actual.isMaybeFloat()) { + if (!f.encoder().writeOp(Op::F64PromoteF32)) { + return false; + } + } else if (actual.isSigned()) { + if (!f.encoder().writeOp(Op::F64ConvertI32S)) { + return false; + } + } else if (actual.isUnsigned()) { + if (!f.encoder().writeOp(Op::F64ConvertI32U)) { + return false; + } + } else { + return f.failf( + expr, "%s is not a subtype of double?, float?, signed or unsigned", + actual.toChars()); + } + break; + default: + MOZ_CRASH("unexpected uncoerced result type"); + } + + *type = Type::ret(expected); + return true; +} + +template +static bool CheckCoercedMathBuiltinCall(FunctionValidator& f, + ParseNode* callNode, + AsmJSMathBuiltinFunction func, Type ret, + Type* type) { + Type actual; + if (!CheckMathBuiltinCall(f, callNode, func, &actual)) { + return false; + } + return CoerceResult(f, callNode, ret, actual, type); +} + +template +static bool CheckCoercedCall(FunctionValidator& f, ParseNode* call, + Type ret, Type* type) { + MOZ_ASSERT(ret.isCanonical()); + + AutoCheckRecursionLimit recursion(f.fc()); + if (!recursion.checkDontReport(f.fc())) { + return f.m().failOverRecursed(); + } + + if (IsNumericLiteral(f.m(), call)) { + NumLit lit = ExtractNumericLiteral(f.m(), call); + if (!f.writeConstExpr(lit)) { + return false; + } + return CoerceResult(f, call, ret, Type::lit(lit), type); + } + + ParseNode* callee = CallCallee(call); + + if (callee->isKind(ParseNodeKind::ElemExpr)) { + return CheckFuncPtrCall(f, call, ret, type); + } + + if (!callee->isKind(ParseNodeKind::Name)) { + return f.fail(callee, "unexpected callee expression type"); + } + + TaggedParserAtomIndex calleeName = callee->as().name(); + + if (const ModuleValidatorShared::Global* global = + f.lookupGlobal(calleeName)) { + switch (global->which()) { + case ModuleValidatorShared::Global::FFI: + return CheckFFICall(f, call, global->ffiIndex(), ret, type); + case ModuleValidatorShared::Global::MathBuiltinFunction: + return CheckCoercedMathBuiltinCall( + f, call, global->mathBuiltinFunction(), ret, type); + case ModuleValidatorShared::Global::ConstantLiteral: + case ModuleValidatorShared::Global::ConstantImport: + case ModuleValidatorShared::Global::Variable: + case ModuleValidatorShared::Global::Table: + case ModuleValidatorShared::Global::ArrayView: + case ModuleValidatorShared::Global::ArrayViewCtor: + return f.failName(callee, "'%s' is not callable function", calleeName); + case ModuleValidatorShared::Global::Function: + break; + } + } + + return CheckInternalCall(f, call, calleeName, ret, type); +} + +template +static bool CheckPos(FunctionValidator& f, ParseNode* pos, Type* type) { + MOZ_ASSERT(pos->isKind(ParseNodeKind::PosExpr)); + ParseNode* operand = UnaryKid(pos); + + if (operand->isKind(ParseNodeKind::CallExpr)) { + return CheckCoercedCall(f, operand, Type::Double, type); + } + + Type actual; + if (!CheckExpr(f, operand, &actual)) { + return false; + } + + return CoerceResult(f, operand, Type::Double, actual, type); +} + +template +static bool CheckNot(FunctionValidator& f, ParseNode* expr, Type* type) { + MOZ_ASSERT(expr->isKind(ParseNodeKind::NotExpr)); + ParseNode* operand = UnaryKid(expr); + + Type operandType; + if (!CheckExpr(f, operand, &operandType)) { + return false; + } + + if (!operandType.isInt()) { + return f.failf(operand, "%s is not a subtype of int", + operandType.toChars()); + } + + *type = Type::Int; + return f.encoder().writeOp(Op::I32Eqz); +} + +template +static bool CheckNeg(FunctionValidator& f, ParseNode* expr, Type* type) { + MOZ_ASSERT(expr->isKind(ParseNodeKind::NegExpr)); + ParseNode* operand = UnaryKid(expr); + + Type operandType; + if (!CheckExpr(f, operand, &operandType)) { + return false; + } + + if (operandType.isInt()) { + *type = Type::Intish; + return f.encoder().writeOp(MozOp::I32Neg); + } + + if (operandType.isMaybeDouble()) { + *type = Type::Double; + return f.encoder().writeOp(Op::F64Neg); + } + + if (operandType.isMaybeFloat()) { + *type = Type::Floatish; + return f.encoder().writeOp(Op::F32Neg); + } + + return f.failf(operand, "%s is not a subtype of int, float? or double?", + operandType.toChars()); +} + +template +static bool CheckCoerceToInt(FunctionValidator& f, ParseNode* expr, + Type* type) { + MOZ_ASSERT(expr->isKind(ParseNodeKind::BitNotExpr)); + ParseNode* operand = UnaryKid(expr); + + Type operandType; + if (!CheckExpr(f, operand, &operandType)) { + return false; + } + + if (operandType.isMaybeDouble() || operandType.isMaybeFloat()) { + *type = Type::Signed; + Op opcode = + operandType.isMaybeDouble() ? Op::I32TruncF64S : Op::I32TruncF32S; + return f.encoder().writeOp(opcode); + } + + if (!operandType.isIntish()) { + return f.failf(operand, "%s is not a subtype of double?, float? or intish", + operandType.toChars()); + } + + *type = Type::Signed; + return true; +} + +template +static bool CheckBitNot(FunctionValidator& f, ParseNode* neg, + Type* type) { + MOZ_ASSERT(neg->isKind(ParseNodeKind::BitNotExpr)); + ParseNode* operand = UnaryKid(neg); + + if (operand->isKind(ParseNodeKind::BitNotExpr)) { + return CheckCoerceToInt(f, operand, type); + } + + Type operandType; + if (!CheckExpr(f, operand, &operandType)) { + return false; + } + + if (!operandType.isIntish()) { + return f.failf(operand, "%s is not a subtype of intish", + operandType.toChars()); + } + + if (!f.encoder().writeOp(MozOp::I32BitNot)) { + return false; + } + + *type = Type::Signed; + return true; +} + +template +static bool CheckAsExprStatement(FunctionValidator& f, + ParseNode* exprStmt); + +template +static bool CheckComma(FunctionValidator& f, ParseNode* comma, + Type* type) { + MOZ_ASSERT(comma->isKind(ParseNodeKind::CommaExpr)); + ParseNode* operands = ListHead(comma); + + // The block depth isn't taken into account here, because a comma list can't + // contain breaks and continues and nested control flow structures. + if (!f.encoder().writeOp(Op::Block)) { + return false; + } + + size_t typeAt; + if (!f.encoder().writePatchableFixedU7(&typeAt)) { + return false; + } + + ParseNode* pn = operands; + for (; NextNode(pn); pn = NextNode(pn)) { + if (!CheckAsExprStatement(f, pn)) { + return false; + } + } + + if (!CheckExpr(f, pn, type)) { + return false; + } + + f.encoder().patchFixedU7(typeAt, uint8_t(type->toWasmBlockSignatureType())); + + return f.encoder().writeOp(Op::End); +} + +template +static bool CheckConditional(FunctionValidator& f, ParseNode* ternary, + Type* type) { + MOZ_ASSERT(ternary->isKind(ParseNodeKind::ConditionalExpr)); + + ParseNode* cond = TernaryKid1(ternary); + ParseNode* thenExpr = TernaryKid2(ternary); + ParseNode* elseExpr = TernaryKid3(ternary); + + Type condType; + if (!CheckExpr(f, cond, &condType)) { + return false; + } + + if (!condType.isInt()) { + return f.failf(cond, "%s is not a subtype of int", condType.toChars()); + } + + size_t typeAt; + if (!f.pushIf(&typeAt)) { + return false; + } + + Type thenType; + if (!CheckExpr(f, thenExpr, &thenType)) { + return false; + } + + if (!f.switchToElse()) { + return false; + } + + Type elseType; + if (!CheckExpr(f, elseExpr, &elseType)) { + return false; + } + + if (thenType.isInt() && elseType.isInt()) { + *type = Type::Int; + } else if (thenType.isDouble() && elseType.isDouble()) { + *type = Type::Double; + } else if (thenType.isFloat() && elseType.isFloat()) { + *type = Type::Float; + } else { + return f.failf( + ternary, + "then/else branches of conditional must both produce int, float, " + "double, current types are %s and %s", + thenType.toChars(), elseType.toChars()); + } + + if (!f.popIf(typeAt, type->toWasmBlockSignatureType())) { + return false; + } + + return true; +} + +template +static bool IsValidIntMultiplyConstant(ModuleValidator& m, + ParseNode* expr) { + if (!IsNumericLiteral(m, expr)) { + return false; + } + + NumLit lit = ExtractNumericLiteral(m, expr); + switch (lit.which()) { + case NumLit::Fixnum: + case NumLit::NegativeInt: + if (Abs(lit.toInt32()) < (uint32_t(1) << 20)) { + return true; + } + return false; + case NumLit::BigUnsigned: + case NumLit::Double: + case NumLit::Float: + case NumLit::OutOfRangeInt: + return false; + } + + MOZ_CRASH("Bad literal"); +} + +template +static bool CheckMultiply(FunctionValidator& f, ParseNode* star, + Type* type) { + MOZ_ASSERT(star->isKind(ParseNodeKind::MulExpr)); + ParseNode* lhs = MultiplyLeft(star); + ParseNode* rhs = MultiplyRight(star); + + Type lhsType; + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + if (lhsType.isInt() && rhsType.isInt()) { + if (!IsValidIntMultiplyConstant(f.m(), lhs) && + !IsValidIntMultiplyConstant(f.m(), rhs)) { + return f.fail( + star, + "one arg to int multiply must be a small (-2^20, 2^20) int literal"); + } + *type = Type::Intish; + return f.encoder().writeOp(Op::I32Mul); + } + + if (lhsType.isMaybeDouble() && rhsType.isMaybeDouble()) { + *type = Type::Double; + return f.encoder().writeOp(Op::F64Mul); + } + + if (lhsType.isMaybeFloat() && rhsType.isMaybeFloat()) { + *type = Type::Floatish; + return f.encoder().writeOp(Op::F32Mul); + } + + return f.fail( + star, "multiply operands must be both int, both double? or both float?"); +} + +template +static bool CheckAddOrSub(FunctionValidator& f, ParseNode* expr, + Type* type, unsigned* numAddOrSubOut = nullptr) { + AutoCheckRecursionLimit recursion(f.fc()); + if (!recursion.checkDontReport(f.fc())) { + return f.m().failOverRecursed(); + } + + MOZ_ASSERT(expr->isKind(ParseNodeKind::AddExpr) || + expr->isKind(ParseNodeKind::SubExpr)); + ParseNode* lhs = AddSubLeft(expr); + ParseNode* rhs = AddSubRight(expr); + + Type lhsType, rhsType; + unsigned lhsNumAddOrSub, rhsNumAddOrSub; + + if (lhs->isKind(ParseNodeKind::AddExpr) || + lhs->isKind(ParseNodeKind::SubExpr)) { + if (!CheckAddOrSub(f, lhs, &lhsType, &lhsNumAddOrSub)) { + return false; + } + if (lhsType == Type::Intish) { + lhsType = Type::Int; + } + } else { + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + lhsNumAddOrSub = 0; + } + + if (rhs->isKind(ParseNodeKind::AddExpr) || + rhs->isKind(ParseNodeKind::SubExpr)) { + if (!CheckAddOrSub(f, rhs, &rhsType, &rhsNumAddOrSub)) { + return false; + } + if (rhsType == Type::Intish) { + rhsType = Type::Int; + } + } else { + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + rhsNumAddOrSub = 0; + } + + unsigned numAddOrSub = lhsNumAddOrSub + rhsNumAddOrSub + 1; + if (numAddOrSub > (1 << 20)) { + return f.fail(expr, "too many + or - without intervening coercion"); + } + + if (lhsType.isInt() && rhsType.isInt()) { + if (!f.encoder().writeOp( + expr->isKind(ParseNodeKind::AddExpr) ? Op::I32Add : Op::I32Sub)) { + return false; + } + *type = Type::Intish; + } else if (lhsType.isMaybeDouble() && rhsType.isMaybeDouble()) { + if (!f.encoder().writeOp( + expr->isKind(ParseNodeKind::AddExpr) ? Op::F64Add : Op::F64Sub)) { + return false; + } + *type = Type::Double; + } else if (lhsType.isMaybeFloat() && rhsType.isMaybeFloat()) { + if (!f.encoder().writeOp( + expr->isKind(ParseNodeKind::AddExpr) ? Op::F32Add : Op::F32Sub)) { + return false; + } + *type = Type::Floatish; + } else { + return f.failf( + expr, + "operands to + or - must both be int, float? or double?, got %s and %s", + lhsType.toChars(), rhsType.toChars()); + } + + if (numAddOrSubOut) { + *numAddOrSubOut = numAddOrSub; + } + return true; +} + +template +static bool CheckDivOrMod(FunctionValidator& f, ParseNode* expr, + Type* type) { + MOZ_ASSERT(expr->isKind(ParseNodeKind::DivExpr) || + expr->isKind(ParseNodeKind::ModExpr)); + + ParseNode* lhs = DivOrModLeft(expr); + ParseNode* rhs = DivOrModRight(expr); + + Type lhsType, rhsType; + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + if (lhsType.isMaybeDouble() && rhsType.isMaybeDouble()) { + *type = Type::Double; + if (expr->isKind(ParseNodeKind::DivExpr)) { + return f.encoder().writeOp(Op::F64Div); + } + return f.encoder().writeOp(MozOp::F64Mod); + } + + if (lhsType.isMaybeFloat() && rhsType.isMaybeFloat()) { + *type = Type::Floatish; + if (expr->isKind(ParseNodeKind::DivExpr)) { + return f.encoder().writeOp(Op::F32Div); + } + return f.fail(expr, "modulo cannot receive float arguments"); + } + + if (lhsType.isSigned() && rhsType.isSigned()) { + *type = Type::Intish; + return f.encoder().writeOp( + expr->isKind(ParseNodeKind::DivExpr) ? Op::I32DivS : Op::I32RemS); + } + + if (lhsType.isUnsigned() && rhsType.isUnsigned()) { + *type = Type::Intish; + return f.encoder().writeOp( + expr->isKind(ParseNodeKind::DivExpr) ? Op::I32DivU : Op::I32RemU); + } + + return f.failf( + expr, + "arguments to / or %% must both be double?, float?, signed, or unsigned; " + "%s and %s are given", + lhsType.toChars(), rhsType.toChars()); +} + +template +static bool CheckComparison(FunctionValidator& f, ParseNode* comp, + Type* type) { + MOZ_ASSERT(comp->isKind(ParseNodeKind::LtExpr) || + comp->isKind(ParseNodeKind::LeExpr) || + comp->isKind(ParseNodeKind::GtExpr) || + comp->isKind(ParseNodeKind::GeExpr) || + comp->isKind(ParseNodeKind::EqExpr) || + comp->isKind(ParseNodeKind::NeExpr)); + + ParseNode* lhs = ComparisonLeft(comp); + ParseNode* rhs = ComparisonRight(comp); + + Type lhsType, rhsType; + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + if (!(lhsType.isSigned() && rhsType.isSigned()) && + !(lhsType.isUnsigned() && rhsType.isUnsigned()) && + !(lhsType.isDouble() && rhsType.isDouble()) && + !(lhsType.isFloat() && rhsType.isFloat())) { + return f.failf(comp, + "arguments to a comparison must both be signed, unsigned, " + "floats or doubles; " + "%s and %s are given", + lhsType.toChars(), rhsType.toChars()); + } + + Op stmt; + if (lhsType.isSigned() && rhsType.isSigned()) { + switch (comp->getKind()) { + case ParseNodeKind::EqExpr: + stmt = Op::I32Eq; + break; + case ParseNodeKind::NeExpr: + stmt = Op::I32Ne; + break; + case ParseNodeKind::LtExpr: + stmt = Op::I32LtS; + break; + case ParseNodeKind::LeExpr: + stmt = Op::I32LeS; + break; + case ParseNodeKind::GtExpr: + stmt = Op::I32GtS; + break; + case ParseNodeKind::GeExpr: + stmt = Op::I32GeS; + break; + default: + MOZ_CRASH("unexpected comparison op"); + } + } else if (lhsType.isUnsigned() && rhsType.isUnsigned()) { + switch (comp->getKind()) { + case ParseNodeKind::EqExpr: + stmt = Op::I32Eq; + break; + case ParseNodeKind::NeExpr: + stmt = Op::I32Ne; + break; + case ParseNodeKind::LtExpr: + stmt = Op::I32LtU; + break; + case ParseNodeKind::LeExpr: + stmt = Op::I32LeU; + break; + case ParseNodeKind::GtExpr: + stmt = Op::I32GtU; + break; + case ParseNodeKind::GeExpr: + stmt = Op::I32GeU; + break; + default: + MOZ_CRASH("unexpected comparison op"); + } + } else if (lhsType.isDouble()) { + switch (comp->getKind()) { + case ParseNodeKind::EqExpr: + stmt = Op::F64Eq; + break; + case ParseNodeKind::NeExpr: + stmt = Op::F64Ne; + break; + case ParseNodeKind::LtExpr: + stmt = Op::F64Lt; + break; + case ParseNodeKind::LeExpr: + stmt = Op::F64Le; + break; + case ParseNodeKind::GtExpr: + stmt = Op::F64Gt; + break; + case ParseNodeKind::GeExpr: + stmt = Op::F64Ge; + break; + default: + MOZ_CRASH("unexpected comparison op"); + } + } else if (lhsType.isFloat()) { + switch (comp->getKind()) { + case ParseNodeKind::EqExpr: + stmt = Op::F32Eq; + break; + case ParseNodeKind::NeExpr: + stmt = Op::F32Ne; + break; + case ParseNodeKind::LtExpr: + stmt = Op::F32Lt; + break; + case ParseNodeKind::LeExpr: + stmt = Op::F32Le; + break; + case ParseNodeKind::GtExpr: + stmt = Op::F32Gt; + break; + case ParseNodeKind::GeExpr: + stmt = Op::F32Ge; + break; + default: + MOZ_CRASH("unexpected comparison op"); + } + } else { + MOZ_CRASH("unexpected type"); + } + + *type = Type::Int; + return f.encoder().writeOp(stmt); +} + +template +static bool CheckBitwise(FunctionValidator& f, ParseNode* bitwise, + Type* type) { + ParseNode* lhs = BitwiseLeft(bitwise); + ParseNode* rhs = BitwiseRight(bitwise); + + int32_t identityElement; + bool onlyOnRight; + switch (bitwise->getKind()) { + case ParseNodeKind::BitOrExpr: + identityElement = 0; + onlyOnRight = false; + *type = Type::Signed; + break; + case ParseNodeKind::BitAndExpr: + identityElement = -1; + onlyOnRight = false; + *type = Type::Signed; + break; + case ParseNodeKind::BitXorExpr: + identityElement = 0; + onlyOnRight = false; + *type = Type::Signed; + break; + case ParseNodeKind::LshExpr: + identityElement = 0; + onlyOnRight = true; + *type = Type::Signed; + break; + case ParseNodeKind::RshExpr: + identityElement = 0; + onlyOnRight = true; + *type = Type::Signed; + break; + case ParseNodeKind::UrshExpr: + identityElement = 0; + onlyOnRight = true; + *type = Type::Unsigned; + break; + default: + MOZ_CRASH("not a bitwise op"); + } + + uint32_t i; + if (!onlyOnRight && IsLiteralInt(f.m(), lhs, &i) && + i == uint32_t(identityElement)) { + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + if (!rhsType.isIntish()) { + return f.failf(bitwise, "%s is not a subtype of intish", + rhsType.toChars()); + } + return true; + } + + if (IsLiteralInt(f.m(), rhs, &i) && i == uint32_t(identityElement)) { + if (bitwise->isKind(ParseNodeKind::BitOrExpr) && + lhs->isKind(ParseNodeKind::CallExpr)) { + return CheckCoercedCall(f, lhs, Type::Int, type); + } + + Type lhsType; + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + if (!lhsType.isIntish()) { + return f.failf(bitwise, "%s is not a subtype of intish", + lhsType.toChars()); + } + return true; + } + + Type lhsType; + if (!CheckExpr(f, lhs, &lhsType)) { + return false; + } + + Type rhsType; + if (!CheckExpr(f, rhs, &rhsType)) { + return false; + } + + if (!lhsType.isIntish()) { + return f.failf(lhs, "%s is not a subtype of intish", lhsType.toChars()); + } + if (!rhsType.isIntish()) { + return f.failf(rhs, "%s is not a subtype of intish", rhsType.toChars()); + } + + switch (bitwise->getKind()) { + case ParseNodeKind::BitOrExpr: + if (!f.encoder().writeOp(Op::I32Or)) return false; + break; + case ParseNodeKind::BitAndExpr: + if (!f.encoder().writeOp(Op::I32And)) return false; + break; + case ParseNodeKind::BitXorExpr: + if (!f.encoder().writeOp(Op::I32Xor)) return false; + break; + case ParseNodeKind::LshExpr: + if (!f.encoder().writeOp(Op::I32Shl)) return false; + break; + case ParseNodeKind::RshExpr: + if (!f.encoder().writeOp(Op::I32ShrS)) return false; + break; + case ParseNodeKind::UrshExpr: + if (!f.encoder().writeOp(Op::I32ShrU)) return false; + break; + default: + MOZ_CRASH("not a bitwise op"); + } + + return true; +} + +template +static bool CheckExpr(FunctionValidator& f, ParseNode* expr, Type* type) { + AutoCheckRecursionLimit recursion(f.fc()); + if (!recursion.checkDontReport(f.fc())) { + return f.m().failOverRecursed(); + } + + if (IsNumericLiteral(f.m(), expr)) { + return CheckNumericLiteral(f, expr, type); + } + + switch (expr->getKind()) { + case ParseNodeKind::Name: + return CheckVarRef(f, expr, type); + case ParseNodeKind::ElemExpr: + return CheckLoadArray(f, expr, type); + case ParseNodeKind::AssignExpr: + return CheckAssign(f, expr, type); + case ParseNodeKind::PosExpr: + return CheckPos(f, expr, type); + case ParseNodeKind::NotExpr: + return CheckNot(f, expr, type); + case ParseNodeKind::NegExpr: + return CheckNeg(f, expr, type); + case ParseNodeKind::BitNotExpr: + return CheckBitNot(f, expr, type); + case ParseNodeKind::CommaExpr: + return CheckComma(f, expr, type); + case ParseNodeKind::ConditionalExpr: + return CheckConditional(f, expr, type); + case ParseNodeKind::MulExpr: + return CheckMultiply(f, expr, type); + case ParseNodeKind::CallExpr: + return CheckUncoercedCall(f, expr, type); + + case ParseNodeKind::AddExpr: + case ParseNodeKind::SubExpr: + return CheckAddOrSub(f, expr, type); + + case ParseNodeKind::DivExpr: + case ParseNodeKind::ModExpr: + return CheckDivOrMod(f, expr, type); + + case ParseNodeKind::LtExpr: + case ParseNodeKind::LeExpr: + case ParseNodeKind::GtExpr: + case ParseNodeKind::GeExpr: + case ParseNodeKind::EqExpr: + case ParseNodeKind::NeExpr: + return CheckComparison(f, expr, type); + + case ParseNodeKind::BitOrExpr: + case ParseNodeKind::BitAndExpr: + case ParseNodeKind::BitXorExpr: + case ParseNodeKind::LshExpr: + case ParseNodeKind::RshExpr: + case ParseNodeKind::UrshExpr: + return CheckBitwise(f, expr, type); + + default:; + } + + return f.fail(expr, "unsupported expression"); +} + +template +static bool CheckStatement(FunctionValidator& f, ParseNode* stmt); + +template +static bool CheckAsExprStatement(FunctionValidator& f, ParseNode* expr) { + if (expr->isKind(ParseNodeKind::CallExpr)) { + Type ignored; + return CheckCoercedCall(f, expr, Type::Void, &ignored); + } + + Type resultType; + if (!CheckExpr(f, expr, &resultType)) { + return false; + } + + if (!resultType.isVoid()) { + if (!f.encoder().writeOp(Op::Drop)) { + return false; + } + } + + return true; +} + +template +static bool CheckExprStatement(FunctionValidator& f, + ParseNode* exprStmt) { + MOZ_ASSERT(exprStmt->isKind(ParseNodeKind::ExpressionStmt)); + return CheckAsExprStatement(f, UnaryKid(exprStmt)); +} + +template +static bool CheckLoopConditionOnEntry(FunctionValidator& f, + ParseNode* cond) { + uint32_t maybeLit; + if (IsLiteralInt(f.m(), cond, &maybeLit) && maybeLit) { + return true; + } + + Type condType; + if (!CheckExpr(f, cond, &condType)) { + return false; + } + if (!condType.isInt()) { + return f.failf(cond, "%s is not a subtype of int", condType.toChars()); + } + + if (!f.encoder().writeOp(Op::I32Eqz)) { + return false; + } + + // brIf (i32.eqz $f) $out + if (!f.writeBreakIf()) { + return false; + } + + return true; +} + +template +static bool CheckWhile(FunctionValidator& f, ParseNode* whileStmt, + const LabelVector* labels = nullptr) { + MOZ_ASSERT(whileStmt->isKind(ParseNodeKind::WhileStmt)); + ParseNode* cond = BinaryLeft(whileStmt); + ParseNode* body = BinaryRight(whileStmt); + + // A while loop `while(#cond) #body` is equivalent to: + // (block $after_loop + // (loop $top + // (brIf $after_loop (i32.eq 0 #cond)) + // #body + // (br $top) + // ) + // ) + if (labels && !f.addLabels(*labels, 0, 1)) { + return false; + } + + if (!f.pushLoop()) { + return false; + } + + if (!CheckLoopConditionOnEntry(f, cond)) { + return false; + } + if (!CheckStatement(f, body)) { + return false; + } + if (!f.writeContinue()) { + return false; + } + + if (!f.popLoop()) { + return false; + } + if (labels) { + f.removeLabels(*labels); + } + return true; +} + +template +static bool CheckFor(FunctionValidator& f, ParseNode* forStmt, + const LabelVector* labels = nullptr) { + MOZ_ASSERT(forStmt->isKind(ParseNodeKind::ForStmt)); + ParseNode* forHead = BinaryLeft(forStmt); + ParseNode* body = BinaryRight(forStmt); + + if (!forHead->isKind(ParseNodeKind::ForHead)) { + return f.fail(forHead, "unsupported for-loop statement"); + } + + ParseNode* maybeInit = TernaryKid1(forHead); + ParseNode* maybeCond = TernaryKid2(forHead); + ParseNode* maybeInc = TernaryKid3(forHead); + + // A for-loop `for (#init; #cond; #inc) #body` is equivalent to: + // (block // depth X + // (#init) + // (block $after_loop // depth X+1 (block) + // (loop $loop_top // depth X+2 (loop) + // (brIf $after (eq 0 #cond)) + // (block $after_body #body) // depth X+3 + // #inc + // (br $loop_top) + // ) + // ) + // ) + // A break in the body should break out to $after_loop, i.e. depth + 1. + // A continue in the body should break out to $after_body, i.e. depth + 3. + if (labels && !f.addLabels(*labels, 1, 3)) { + return false; + } + + if (!f.pushUnbreakableBlock()) { + return false; + } + + if (maybeInit && !CheckAsExprStatement(f, maybeInit)) { + return false; + } + + { + if (!f.pushLoop()) { + return false; + } + + if (maybeCond && !CheckLoopConditionOnEntry(f, maybeCond)) { + return false; + } + + { + // Continuing in the body should just break out to the increment. + if (!f.pushContinuableBlock()) { + return false; + } + if (!CheckStatement(f, body)) { + return false; + } + if (!f.popContinuableBlock()) { + return false; + } + } + + if (maybeInc && !CheckAsExprStatement(f, maybeInc)) { + return false; + } + + if (!f.writeContinue()) { + return false; + } + if (!f.popLoop()) { + return false; + } + } + + if (!f.popUnbreakableBlock()) { + return false; + } + + if (labels) { + f.removeLabels(*labels); + } + + return true; +} + +template +static bool CheckDoWhile(FunctionValidator& f, ParseNode* whileStmt, + const LabelVector* labels = nullptr) { + MOZ_ASSERT(whileStmt->isKind(ParseNodeKind::DoWhileStmt)); + ParseNode* body = BinaryLeft(whileStmt); + ParseNode* cond = BinaryRight(whileStmt); + + // A do-while loop `do { #body } while (#cond)` is equivalent to: + // (block $after_loop // depth X + // (loop $top // depth X+1 + // (block #body) // depth X+2 + // (brIf #cond $top) + // ) + // ) + // A break should break out of the entire loop, i.e. at depth 0. + // A continue should break out to the condition, i.e. at depth 2. + if (labels && !f.addLabels(*labels, 0, 2)) { + return false; + } + + if (!f.pushLoop()) { + return false; + } + + { + // An unlabeled continue in the body should break out to the condition. + if (!f.pushContinuableBlock()) { + return false; + } + if (!CheckStatement(f, body)) { + return false; + } + if (!f.popContinuableBlock()) { + return false; + } + } + + Type condType; + if (!CheckExpr(f, cond, &condType)) { + return false; + } + if (!condType.isInt()) { + return f.failf(cond, "%s is not a subtype of int", condType.toChars()); + } + + if (!f.writeContinueIf()) { + return false; + } + + if (!f.popLoop()) { + return false; + } + if (labels) { + f.removeLabels(*labels); + } + return true; +} + +template +static bool CheckStatementList(FunctionValidator& f, ParseNode*, + const LabelVector* = nullptr); + +template +static bool CheckLabel(FunctionValidator& f, ParseNode* labeledStmt) { + MOZ_ASSERT(labeledStmt->isKind(ParseNodeKind::LabelStmt)); + + LabelVector labels; + ParseNode* innermost = labeledStmt; + do { + if (!labels.append(LabeledStatementLabel(innermost))) { + return false; + } + innermost = LabeledStatementStatement(innermost); + } while (innermost->getKind() == ParseNodeKind::LabelStmt); + + switch (innermost->getKind()) { + case ParseNodeKind::ForStmt: + return CheckFor(f, innermost, &labels); + case ParseNodeKind::DoWhileStmt: + return CheckDoWhile(f, innermost, &labels); + case ParseNodeKind::WhileStmt: + return CheckWhile(f, innermost, &labels); + case ParseNodeKind::StatementList: + return CheckStatementList(f, innermost, &labels); + default: + break; + } + + if (!f.pushUnbreakableBlock(&labels)) { + return false; + } + + if (!CheckStatement(f, innermost)) { + return false; + } + + if (!f.popUnbreakableBlock(&labels)) { + return false; + } + return true; +} + +template +static bool CheckIf(FunctionValidator& f, ParseNode* ifStmt) { + uint32_t numIfEnd = 1; + +recurse: + MOZ_ASSERT(ifStmt->isKind(ParseNodeKind::IfStmt)); + ParseNode* cond = TernaryKid1(ifStmt); + ParseNode* thenStmt = TernaryKid2(ifStmt); + ParseNode* elseStmt = TernaryKid3(ifStmt); + + Type condType; + if (!CheckExpr(f, cond, &condType)) { + return false; + } + if (!condType.isInt()) { + return f.failf(cond, "%s is not a subtype of int", condType.toChars()); + } + + size_t typeAt; + if (!f.pushIf(&typeAt)) { + return false; + } + + f.setIfType(typeAt, TypeCode::BlockVoid); + + if (!CheckStatement(f, thenStmt)) { + return false; + } + + if (elseStmt) { + if (!f.switchToElse()) { + return false; + } + + if (elseStmt->isKind(ParseNodeKind::IfStmt)) { + ifStmt = elseStmt; + if (numIfEnd++ == UINT32_MAX) { + return false; + } + goto recurse; + } + + if (!CheckStatement(f, elseStmt)) { + return false; + } + } + + for (uint32_t i = 0; i != numIfEnd; ++i) { + if (!f.popIf()) { + return false; + } + } + + return true; +} + +static bool CheckCaseExpr(FunctionValidatorShared& f, ParseNode* caseExpr, + int32_t* value) { + if (!IsNumericLiteral(f.m(), caseExpr)) { + return f.fail(caseExpr, + "switch case expression must be an integer literal"); + } + + NumLit lit = ExtractNumericLiteral(f.m(), caseExpr); + switch (lit.which()) { + case NumLit::Fixnum: + case NumLit::NegativeInt: + *value = lit.toInt32(); + break; + case NumLit::OutOfRangeInt: + case NumLit::BigUnsigned: + return f.fail(caseExpr, "switch case expression out of integer range"); + case NumLit::Double: + case NumLit::Float: + return f.fail(caseExpr, + "switch case expression must be an integer literal"); + } + + return true; +} + +static bool CheckDefaultAtEnd(FunctionValidatorShared& f, ParseNode* stmt) { + for (; stmt; stmt = NextNode(stmt)) { + if (IsDefaultCase(stmt) && NextNode(stmt) != nullptr) { + return f.fail(stmt, "default label must be at the end"); + } + } + + return true; +} + +static bool CheckSwitchRange(FunctionValidatorShared& f, ParseNode* stmt, + int32_t* low, int32_t* high, + uint32_t* tableLength) { + if (IsDefaultCase(stmt)) { + *low = 0; + *high = -1; + *tableLength = 0; + return true; + } + + int32_t i = 0; + if (!CheckCaseExpr(f, CaseExpr(stmt), &i)) { + return false; + } + + *low = *high = i; + + ParseNode* initialStmt = stmt; + for (stmt = NextNode(stmt); stmt && !IsDefaultCase(stmt); + stmt = NextNode(stmt)) { + int32_t i = 0; + if (!CheckCaseExpr(f, CaseExpr(stmt), &i)) { + return false; + } + + *low = std::min(*low, i); + *high = std::max(*high, i); + } + + int64_t i64 = (int64_t(*high) - int64_t(*low)) + 1; + if (i64 > MaxBrTableElems) { + return f.fail( + initialStmt, + "all switch statements generate tables; this table would be too big"); + } + + *tableLength = uint32_t(i64); + return true; +} + +template +static bool CheckSwitchExpr(FunctionValidator& f, ParseNode* switchExpr) { + Type exprType; + if (!CheckExpr(f, switchExpr, &exprType)) { + return false; + } + if (!exprType.isSigned()) { + return f.failf(switchExpr, "%s is not a subtype of signed", + exprType.toChars()); + } + return true; +} + +// A switch will be constructed as: +// - the default block wrapping all the other blocks, to be able to break +// out of the switch with an unlabeled break statement. It has two statements +// (an inner block and the default expr). asm.js rules require default to be at +// the end, so the default block always encloses all the cases blocks. +// - one block per case between low and high; undefined cases just jump to the +// default case. Each of these blocks contain two statements: the next case's +// block and the possibly empty statement list comprising the case body. The +// last block pushed is the first case so the (relative) branch target therefore +// matches the sequential order of cases. +// - one block for the br_table, so that the first break goes to the first +// case's block. +template +static bool CheckSwitch(FunctionValidator& f, ParseNode* switchStmt) { + MOZ_ASSERT(switchStmt->isKind(ParseNodeKind::SwitchStmt)); + + ParseNode* switchExpr = BinaryLeft(switchStmt); + ParseNode* switchBody = BinaryRight(switchStmt); + + if (switchBody->is()) { + LexicalScopeNode* scope = &switchBody->as(); + if (!scope->isEmptyScope()) { + return f.fail(scope, "switch body may not contain lexical declarations"); + } + switchBody = scope->scopeBody(); + } + + ParseNode* stmt = ListHead(switchBody); + if (!stmt) { + if (!CheckSwitchExpr(f, switchExpr)) { + return false; + } + if (!f.encoder().writeOp(Op::Drop)) { + return false; + } + return true; + } + + if (!CheckDefaultAtEnd(f, stmt)) { + return false; + } + + int32_t low = 0, high = 0; + uint32_t tableLength = 0; + if (!CheckSwitchRange(f, stmt, &low, &high, &tableLength)) { + return false; + } + + static const uint32_t CASE_NOT_DEFINED = UINT32_MAX; + + Uint32Vector caseDepths; + if (!caseDepths.appendN(CASE_NOT_DEFINED, tableLength)) { + return false; + } + + uint32_t numCases = 0; + for (ParseNode* s = stmt; s && !IsDefaultCase(s); s = NextNode(s)) { + int32_t caseValue = ExtractNumericLiteral(f.m(), CaseExpr(s)).toInt32(); + + MOZ_ASSERT(caseValue >= low); + unsigned i = caseValue - low; + if (caseDepths[i] != CASE_NOT_DEFINED) { + return f.fail(s, "no duplicate case labels"); + } + + MOZ_ASSERT(numCases != CASE_NOT_DEFINED); + caseDepths[i] = numCases++; + } + + // Open the wrapping breakable default block. + if (!f.pushBreakableBlock()) { + return false; + } + + // Open all the case blocks. + for (uint32_t i = 0; i < numCases; i++) { + if (!f.pushUnbreakableBlock()) { + return false; + } + } + + // Open the br_table block. + if (!f.pushUnbreakableBlock()) { + return false; + } + + // The default block is the last one. + uint32_t defaultDepth = numCases; + + // Subtract lowest case value, so that all the cases start from 0. + if (low) { + if (!CheckSwitchExpr(f, switchExpr)) { + return false; + } + if (!f.writeInt32Lit(low)) { + return false; + } + if (!f.encoder().writeOp(Op::I32Sub)) { + return false; + } + } else { + if (!CheckSwitchExpr(f, switchExpr)) { + return false; + } + } + + // Start the br_table block. + if (!f.encoder().writeOp(Op::BrTable)) { + return false; + } + + // Write the number of cases (tableLength - 1 + 1 (default)). + // Write the number of cases (tableLength - 1 + 1 (default)). + if (!f.encoder().writeVarU32(tableLength)) { + return false; + } + + // Each case value describes the relative depth to the actual block. When + // a case is not explicitly defined, it goes to the default. + for (size_t i = 0; i < tableLength; i++) { + uint32_t target = + caseDepths[i] == CASE_NOT_DEFINED ? defaultDepth : caseDepths[i]; + if (!f.encoder().writeVarU32(target)) { + return false; + } + } + + // Write the default depth. + if (!f.encoder().writeVarU32(defaultDepth)) { + return false; + } + + // Our br_table is done. Close its block, write the cases down in order. + if (!f.popUnbreakableBlock()) { + return false; + } + + for (; stmt && !IsDefaultCase(stmt); stmt = NextNode(stmt)) { + if (!CheckStatement(f, CaseBody(stmt))) { + return false; + } + if (!f.popUnbreakableBlock()) { + return false; + } + } + + // Write the default block. + if (stmt && IsDefaultCase(stmt)) { + if (!CheckStatement(f, CaseBody(stmt))) { + return false; + } + } + + // Close the wrapping block. + return f.popBreakableBlock(); +} + +static bool CheckReturnType(FunctionValidatorShared& f, ParseNode* usepn, + Type ret) { + Maybe type = ret.canonicalToReturnType(); + + if (!f.hasAlreadyReturned()) { + f.setReturnedType(type); + return true; + } + + if (f.returnedType() != type) { + return f.failf(usepn, "%s incompatible with previous return of type %s", + ToString(type, nullptr).get(), + ToString(f.returnedType(), nullptr).get()); + } + + return true; +} + +template +static bool CheckReturn(FunctionValidator& f, ParseNode* returnStmt) { + ParseNode* expr = ReturnExpr(returnStmt); + + if (!expr) { + if (!CheckReturnType(f, returnStmt, Type::Void)) { + return false; + } + } else { + Type type; + if (!CheckExpr(f, expr, &type)) { + return false; + } + + if (!type.isReturnType()) { + return f.failf(expr, "%s is not a valid return type", type.toChars()); + } + + if (!CheckReturnType(f, expr, Type::canonicalize(type))) { + return false; + } + } + + return f.encoder().writeOp(Op::Return); +} + +template +static bool CheckStatementList(FunctionValidator& f, ParseNode* stmtList, + const LabelVector* labels /*= nullptr */) { + MOZ_ASSERT(stmtList->isKind(ParseNodeKind::StatementList)); + + if (!f.pushUnbreakableBlock(labels)) { + return false; + } + + for (ParseNode* stmt = ListHead(stmtList); stmt; stmt = NextNode(stmt)) { + if (!CheckStatement(f, stmt)) { + return false; + } + } + + return f.popUnbreakableBlock(labels); +} + +template +static bool CheckLexicalScope(FunctionValidator& f, ParseNode* node) { + LexicalScopeNode* lexicalScope = &node->as(); + if (!lexicalScope->isEmptyScope()) { + return f.fail(lexicalScope, "cannot have 'let' or 'const' declarations"); + } + + return CheckStatement(f, lexicalScope->scopeBody()); +} + +static bool CheckBreakOrContinue(FunctionValidatorShared& f, bool isBreak, + ParseNode* stmt) { + if (TaggedParserAtomIndex maybeLabel = LoopControlMaybeLabel(stmt)) { + return f.writeLabeledBreakOrContinue(maybeLabel, isBreak); + } + return f.writeUnlabeledBreakOrContinue(isBreak); +} + +template +static bool CheckStatement(FunctionValidator& f, ParseNode* stmt) { + AutoCheckRecursionLimit recursion(f.fc()); + if (!recursion.checkDontReport(f.fc())) { + return f.m().failOverRecursed(); + } + + switch (stmt->getKind()) { + case ParseNodeKind::EmptyStmt: + return true; + case ParseNodeKind::ExpressionStmt: + return CheckExprStatement(f, stmt); + case ParseNodeKind::WhileStmt: + return CheckWhile(f, stmt); + case ParseNodeKind::ForStmt: + return CheckFor(f, stmt); + case ParseNodeKind::DoWhileStmt: + return CheckDoWhile(f, stmt); + case ParseNodeKind::LabelStmt: + return CheckLabel(f, stmt); + case ParseNodeKind::IfStmt: + return CheckIf(f, stmt); + case ParseNodeKind::SwitchStmt: + return CheckSwitch(f, stmt); + case ParseNodeKind::ReturnStmt: + return CheckReturn(f, stmt); + case ParseNodeKind::StatementList: + return CheckStatementList(f, stmt); + case ParseNodeKind::BreakStmt: + return CheckBreakOrContinue(f, true, stmt); + case ParseNodeKind::ContinueStmt: + return CheckBreakOrContinue(f, false, stmt); + case ParseNodeKind::LexicalScope: + return CheckLexicalScope(f, stmt); + default:; + } + + return f.fail(stmt, "unexpected statement kind"); +} + +template +static bool ParseFunction(ModuleValidator& m, FunctionNode** funNodeOut, + unsigned* line) { + auto& tokenStream = m.tokenStream(); + + tokenStream.consumeKnownToken(TokenKind::Function, + TokenStreamShared::SlashIsRegExp); + + auto& anyChars = tokenStream.anyCharsAccess(); + uint32_t toStringStart = anyChars.currentToken().pos.begin; + *line = anyChars.lineNumber(anyChars.lineToken(toStringStart)); + + TokenKind tk; + if (!tokenStream.getToken(&tk, TokenStreamShared::SlashIsRegExp)) { + return false; + } + if (tk == TokenKind::Mul) { + return m.failCurrentOffset("unexpected generator function"); + } + if (!TokenKindIsPossibleIdentifier(tk)) { + return false; // The regular parser will throw a SyntaxError, no need to + // m.fail. + } + + TaggedParserAtomIndex name = m.parser().bindingIdentifier(YieldIsName); + if (!name) { + return false; + } + + FunctionNode* funNode = m.parser().handler_.newFunction( + FunctionSyntaxKind::Statement, m.parser().pos()); + if (!funNode) { + return false; + } + + ParseContext* outerpc = m.parser().pc_; + Directives directives(outerpc); + FunctionFlags flags(FunctionFlags::INTERPRETED_NORMAL); + FunctionBox* funbox = m.parser().newFunctionBox( + funNode, name, flags, toStringStart, directives, + GeneratorKind::NotGenerator, FunctionAsyncKind::SyncFunction); + if (!funbox) { + return false; + } + funbox->initWithEnclosingParseContext(outerpc, FunctionSyntaxKind::Statement); + + Directives newDirectives = directives; + SourceParseContext funpc(&m.parser(), funbox, &newDirectives); + if (!funpc.init()) { + return false; + } + + if (!m.parser().functionFormalParametersAndBody( + InAllowed, YieldIsName, &funNode, FunctionSyntaxKind::Statement)) { + if (anyChars.hadError() || directives == newDirectives) { + return false; + } + + return m.fail(funNode, "encountered new directive in function"); + } + + MOZ_ASSERT(!anyChars.hadError()); + MOZ_ASSERT(directives == newDirectives); + + *funNodeOut = funNode; + return true; +} + +template +static bool CheckFunction(ModuleValidator& m) { + // asm.js modules can be quite large when represented as parse trees so pop + // the backing LifoAlloc after parsing/compiling each function. Release the + // parser's lifo memory after the last use of a parse node. + frontend::ParserBase::Mark mark = m.parser().mark(); + auto releaseMark = + mozilla::MakeScopeExit([&m, &mark] { m.parser().release(mark); }); + + FunctionNode* funNode = nullptr; + unsigned line = 0; + if (!ParseFunction(m, &funNode, &line)) { + return false; + } + + if (!CheckFunctionHead(m, funNode)) { + return false; + } + + FunctionValidator f(m, funNode); + + ParseNode* stmtIter = ListHead(FunctionStatementList(funNode)); + + if (!CheckProcessingDirectives(m, &stmtIter)) { + return false; + } + + ValTypeVector args; + if (!CheckArguments(f, &stmtIter, &args)) { + return false; + } + + if (!CheckVariables(f, &stmtIter)) { + return false; + } + + ParseNode* lastNonEmptyStmt = nullptr; + for (; stmtIter; stmtIter = NextNonEmptyStatement(stmtIter)) { + lastNonEmptyStmt = stmtIter; + if (!CheckStatement(f, stmtIter)) { + return false; + } + } + + if (!CheckFinalReturn(f, lastNonEmptyStmt)) { + return false; + } + + ValTypeVector results; + if (f.returnedType()) { + if (!results.append(f.returnedType().ref())) { + return false; + } + } + + FuncType sig(std::move(args), std::move(results)); + + ModuleValidatorShared::Func* func = nullptr; + if (!CheckFunctionSignature(m, funNode, std::move(sig), FunctionName(funNode), + &func)) { + return false; + } + + if (func->defined()) { + return m.failName(funNode, "function '%s' already defined", + FunctionName(funNode)); + } + + f.define(func, line); + + return true; +} + +static bool CheckAllFunctionsDefined(ModuleValidatorShared& m) { + for (unsigned i = 0; i < m.numFuncDefs(); i++) { + const ModuleValidatorShared::Func& f = m.funcDef(i); + if (!f.defined()) { + return m.failNameOffset(f.firstUse(), "missing definition of function %s", + f.name()); + } + } + + return true; +} + +template +static bool CheckFunctions(ModuleValidator& m) { + while (true) { + TokenKind tk; + if (!PeekToken(m.parser(), &tk)) { + return false; + } + + if (tk != TokenKind::Function) { + break; + } + + if (!CheckFunction(m)) { + return false; + } + } + + return CheckAllFunctionsDefined(m); +} + +template +static bool CheckFuncPtrTable(ModuleValidator& m, ParseNode* decl) { + if (!decl->isKind(ParseNodeKind::AssignExpr)) { + return m.fail(decl, "function-pointer table must have initializer"); + } + AssignmentNode* assignNode = &decl->as(); + + ParseNode* var = assignNode->left(); + + if (!var->isKind(ParseNodeKind::Name)) { + return m.fail(var, "function-pointer table name is not a plain name"); + } + + ParseNode* arrayLiteral = assignNode->right(); + + if (!arrayLiteral->isKind(ParseNodeKind::ArrayExpr)) { + return m.fail( + var, "function-pointer table's initializer must be an array literal"); + } + + unsigned length = ListLength(arrayLiteral); + + if (!IsPowerOfTwo(length)) { + return m.failf(arrayLiteral, + "function-pointer table length must be a power of 2 (is %u)", + length); + } + + unsigned mask = length - 1; + + Uint32Vector elemFuncDefIndices; + const FuncType* sig = nullptr; + for (ParseNode* elem = ListHead(arrayLiteral); elem; elem = NextNode(elem)) { + if (!elem->isKind(ParseNodeKind::Name)) { + return m.fail( + elem, "function-pointer table's elements must be names of functions"); + } + + TaggedParserAtomIndex funcName = elem->as().name(); + const ModuleValidatorShared::Func* func = m.lookupFuncDef(funcName); + if (!func) { + return m.fail( + elem, "function-pointer table's elements must be names of functions"); + } + + const FuncType& funcSig = m.env().types->type(func->sigIndex()).funcType(); + if (sig) { + if (!FuncType::strictlyEquals(*sig, funcSig)) { + return m.fail(elem, "all functions in table must have same signature"); + } + } else { + sig = &funcSig; + } + + if (!elemFuncDefIndices.append(func->funcDefIndex())) { + return false; + } + } + + FuncType copy; + if (!copy.clone(*sig)) { + return false; + } + + uint32_t tableIndex; + if (!CheckFuncPtrTableAgainstExisting(m, var, var->as().name(), + std::move(copy), mask, &tableIndex)) { + return false; + } + + if (!m.defineFuncPtrTable(tableIndex, std::move(elemFuncDefIndices))) { + return m.fail(var, "duplicate function-pointer definition"); + } + + return true; +} + +template +static bool CheckFuncPtrTables(ModuleValidator& m) { + while (true) { + ParseNode* varStmt; + if (!ParseVarOrConstStatement(m.parser(), &varStmt)) { + return false; + } + if (!varStmt) { + break; + } + for (ParseNode* var = VarListHead(varStmt); var; var = NextNode(var)) { + if (!CheckFuncPtrTable(m, var)) { + return false; + } + } + } + + for (unsigned i = 0; i < m.numFuncPtrTables(); i++) { + ModuleValidatorShared::Table& table = m.table(i); + if (!table.defined()) { + return m.failNameOffset(table.firstUse(), + "function-pointer table %s wasn't defined", + table.name()); + } + } + + return true; +} + +static bool CheckModuleExportFunction( + ModuleValidatorShared& m, ParseNode* pn, + TaggedParserAtomIndex maybeFieldName = TaggedParserAtomIndex::null()) { + if (!pn->isKind(ParseNodeKind::Name)) { + return m.fail(pn, "expected name of exported function"); + } + + TaggedParserAtomIndex funcName = pn->as().name(); + const ModuleValidatorShared::Func* func = m.lookupFuncDef(funcName); + if (!func) { + return m.failName(pn, "function '%s' not found", funcName); + } + + return m.addExportField(*func, maybeFieldName); +} + +static bool CheckModuleExportObject(ModuleValidatorShared& m, + ParseNode* object) { + MOZ_ASSERT(object->isKind(ParseNodeKind::ObjectExpr)); + + for (ParseNode* pn = ListHead(object); pn; pn = NextNode(pn)) { + if (!IsNormalObjectField(pn)) { + return m.fail(pn, + "only normal object properties may be used in the export " + "object literal"); + } + + TaggedParserAtomIndex fieldName = ObjectNormalFieldName(pn); + + ParseNode* initNode = ObjectNormalFieldInitializer(pn); + if (!initNode->isKind(ParseNodeKind::Name)) { + return m.fail( + initNode, + "initializer of exported object literal must be name of function"); + } + + if (!CheckModuleExportFunction(m, initNode, fieldName)) { + return false; + } + } + + return true; +} + +template +static bool CheckModuleReturn(ModuleValidator& m) { + TokenKind tk; + if (!GetToken(m.parser(), &tk)) { + return false; + } + auto& ts = m.parser().tokenStream; + if (tk != TokenKind::Return) { + return m.failCurrentOffset( + (tk == TokenKind::RightCurly || tk == TokenKind::Eof) + ? "expecting return statement" + : "invalid asm.js. statement"); + } + ts.anyCharsAccess().ungetToken(); + + ParseNode* returnStmt = m.parser().statementListItem(YieldIsName); + if (!returnStmt) { + return false; + } + + ParseNode* returnExpr = ReturnExpr(returnStmt); + if (!returnExpr) { + return m.fail(returnStmt, "export statement must return something"); + } + + if (returnExpr->isKind(ParseNodeKind::ObjectExpr)) { + if (!CheckModuleExportObject(m, returnExpr)) { + return false; + } + } else { + if (!CheckModuleExportFunction(m, returnExpr)) { + return false; + } + } + + return true; +} + +template +static bool CheckModuleEnd(ModuleValidator& m) { + TokenKind tk; + if (!GetToken(m.parser(), &tk)) { + return false; + } + + if (tk != TokenKind::Eof && tk != TokenKind::RightCurly) { + return m.failCurrentOffset( + "top-level export (return) must be the last statement"); + } + + m.parser().tokenStream.anyCharsAccess().ungetToken(); + return true; +} + +template +static SharedModule CheckModule(FrontendContext* fc, + ParserAtomsTable& parserAtoms, + AsmJSParser& parser, ParseNode* stmtList, + unsigned* time) { + int64_t before = PRMJ_Now(); + + FunctionNode* moduleFunctionNode = parser.pc_->functionBox()->functionNode; + + ModuleValidator m(fc, parserAtoms, parser, moduleFunctionNode); + if (!m.init()) { + return nullptr; + } + + if (!CheckFunctionHead(m, moduleFunctionNode)) { + return nullptr; + } + + if (!CheckModuleArguments(m, moduleFunctionNode)) { + return nullptr; + } + + if (!CheckPrecedingStatements(m, stmtList)) { + return nullptr; + } + + if (!CheckModuleProcessingDirectives(m)) { + return nullptr; + } + + if (!CheckModuleGlobals(m)) { + return nullptr; + } + + if (!m.startFunctionBodies()) { + return nullptr; + } + + if (!CheckFunctions(m)) { + return nullptr; + } + + if (!CheckFuncPtrTables(m)) { + return nullptr; + } + + if (!CheckModuleReturn(m)) { + return nullptr; + } + + if (!CheckModuleEnd(m)) { + return nullptr; + } + + SharedModule module = m.finish(); + if (!module) { + return nullptr; + } + + *time = (PRMJ_Now() - before) / PRMJ_USEC_PER_MSEC; + return module; +} + +/*****************************************************************************/ +// Link-time validation + +static bool LinkFail(JSContext* cx, const char* str) { + WarnNumberASCII(cx, JSMSG_USE_ASM_LINK_FAIL, str); + return false; +} + +static bool IsMaybeWrappedScriptedProxy(JSObject* obj) { + JSObject* unwrapped = UncheckedUnwrap(obj); + return unwrapped && IsScriptedProxy(unwrapped); +} + +static bool GetDataProperty(JSContext* cx, HandleValue objVal, + Handle field, MutableHandleValue v) { + if (!objVal.isObject()) { + return LinkFail(cx, "accessing property of non-object"); + } + + RootedObject obj(cx, &objVal.toObject()); + if (IsMaybeWrappedScriptedProxy(obj)) { + return LinkFail(cx, "accessing property of a Proxy"); + } + + RootedId id(cx, AtomToId(field)); + Rooted> desc(cx); + RootedObject holder(cx); + if (!GetPropertyDescriptor(cx, obj, id, &desc, &holder)) { + return false; + } + + if (!desc.isSome()) { + return LinkFail(cx, "property not present on object"); + } + + if (!desc->isDataDescriptor()) { + return LinkFail(cx, "property is not a data property"); + } + + v.set(desc->value()); + return true; +} + +static bool GetDataProperty(JSContext* cx, HandleValue objVal, + const char* fieldChars, MutableHandleValue v) { + Rooted field(cx, + AtomizeUTF8Chars(cx, fieldChars, strlen(fieldChars))); + if (!field) { + return false; + } + + return GetDataProperty(cx, objVal, field, v); +} + +static bool GetDataProperty(JSContext* cx, HandleValue objVal, + const ImmutableTenuredPtr& field, + MutableHandleValue v) { + Handle fieldHandle = field; + return GetDataProperty(cx, objVal, fieldHandle, v); +} + +static bool HasObjectValueOfMethodPure(JSObject* obj, JSContext* cx) { + Value v; + if (!GetPropertyPure(cx, obj, NameToId(cx->names().valueOf), &v)) { + return false; + } + + JSFunction* fun; + if (!IsFunctionObject(v, &fun)) { + return false; + } + + return IsSelfHostedFunctionWithName(fun, cx->names().Object_valueOf); +} + +static bool HasPureCoercion(JSContext* cx, HandleValue v) { + // Ideally, we'd reject all non-primitives, but Emscripten has a bug that + // generates code that passes functions for some imports. To avoid breaking + // all the code that contains this bug, we make an exception for functions + // that don't have user-defined valueOf or toString, for their coercions + // are not observable and coercion via ToNumber/ToInt32 definitely produces + // NaN/0. We should remove this special case later once most apps have been + // built with newer Emscripten. + return v.toObject().is() && + HasNoToPrimitiveMethodPure(&v.toObject(), cx) && + HasObjectValueOfMethodPure(&v.toObject(), cx) && + HasNativeMethodPure(&v.toObject(), cx->names().toString, fun_toString, + cx); +} + +static bool ValidateGlobalVariable(JSContext* cx, const AsmJSGlobal& global, + HandleValue importVal, + Maybe* val) { + switch (global.varInitKind()) { + case AsmJSGlobal::InitConstant: + val->emplace(global.varInitVal()); + return true; + + case AsmJSGlobal::InitImport: { + RootedValue v(cx); + if (!GetDataProperty(cx, importVal, global.field(), &v)) { + return false; + } + + if (!v.isPrimitive() && !HasPureCoercion(cx, v)) { + return LinkFail(cx, "Imported values must be primitives"); + } + + switch (global.varInitImportType().kind()) { + case ValType::I32: { + int32_t i32; + if (!ToInt32(cx, v, &i32)) { + return false; + } + val->emplace(uint32_t(i32)); + return true; + } + case ValType::I64: + MOZ_CRASH("int64"); + case ValType::V128: + MOZ_CRASH("v128"); + case ValType::F32: { + float f; + if (!RoundFloat32(cx, v, &f)) { + return false; + } + val->emplace(f); + return true; + } + case ValType::F64: { + double d; + if (!ToNumber(cx, v, &d)) { + return false; + } + val->emplace(d); + return true; + } + case ValType::Ref: { + MOZ_CRASH("not available in asm.js"); + } + } + } + } + + MOZ_CRASH("unreachable"); +} + +static bool ValidateFFI(JSContext* cx, const AsmJSGlobal& global, + HandleValue importVal, + MutableHandle ffis) { + RootedValue v(cx); + if (!GetDataProperty(cx, importVal, global.field(), &v)) { + return false; + } + + if (!IsFunctionObject(v)) { + return LinkFail(cx, "FFI imports must be functions"); + } + + ffis[global.ffiIndex()].set(&v.toObject().as()); + return true; +} + +static bool ValidateArrayView(JSContext* cx, const AsmJSGlobal& global, + HandleValue globalVal) { + if (!global.field()) { + return true; + } + + if (Scalar::isBigIntType(global.viewType())) { + return LinkFail(cx, "bad typed array constructor"); + } + + RootedValue v(cx); + if (!GetDataProperty(cx, globalVal, global.field(), &v)) { + return false; + } + + bool tac = IsTypedArrayConstructor(v, global.viewType()); + if (!tac) { + return LinkFail(cx, "bad typed array constructor"); + } + + return true; +} + +static InlinableNative ToInlinableNative(AsmJSMathBuiltinFunction func) { + switch (func) { + case AsmJSMathBuiltin_sin: + return InlinableNative::MathSin; + case AsmJSMathBuiltin_cos: + return InlinableNative::MathCos; + case AsmJSMathBuiltin_tan: + return InlinableNative::MathTan; + case AsmJSMathBuiltin_asin: + return InlinableNative::MathASin; + case AsmJSMathBuiltin_acos: + return InlinableNative::MathACos; + case AsmJSMathBuiltin_atan: + return InlinableNative::MathATan; + case AsmJSMathBuiltin_ceil: + return InlinableNative::MathCeil; + case AsmJSMathBuiltin_floor: + return InlinableNative::MathFloor; + case AsmJSMathBuiltin_exp: + return InlinableNative::MathExp; + case AsmJSMathBuiltin_log: + return InlinableNative::MathLog; + case AsmJSMathBuiltin_pow: + return InlinableNative::MathPow; + case AsmJSMathBuiltin_sqrt: + return InlinableNative::MathSqrt; + case AsmJSMathBuiltin_abs: + return InlinableNative::MathAbs; + case AsmJSMathBuiltin_atan2: + return InlinableNative::MathATan2; + case AsmJSMathBuiltin_imul: + return InlinableNative::MathImul; + case AsmJSMathBuiltin_fround: + return InlinableNative::MathFRound; + case AsmJSMathBuiltin_min: + return InlinableNative::MathMin; + case AsmJSMathBuiltin_max: + return InlinableNative::MathMax; + case AsmJSMathBuiltin_clz32: + return InlinableNative::MathClz32; + } + MOZ_CRASH("Invalid asm.js math builtin function"); +} + +static bool ValidateMathBuiltinFunction(JSContext* cx, + const AsmJSMetadata& metadata, + const AsmJSGlobal& global, + HandleValue globalVal) { + RootedValue v(cx); + if (!GetDataProperty(cx, globalVal, cx->names().Math, &v)) { + return false; + } + + if (!GetDataProperty(cx, v, global.field(), &v)) { + return false; + } + + InlinableNative native = ToInlinableNative(global.mathBuiltinFunction()); + + JSFunction* fun; + if (!IsFunctionObject(v, &fun) || !fun->hasJitInfo() || + fun->jitInfo()->type() != JSJitInfo::InlinableNative || + fun->jitInfo()->inlinableNative != native) { + return LinkFail(cx, "bad Math.* builtin function"); + } + if (fun->realm()->behaviors().shouldResistFingerprinting() != + metadata.shouldResistFingerprinting) { + return LinkFail(cx, + "Math.* builtin function and asm.js module have a " + "different resist fingerprinting mode"); + } + + return true; +} + +static bool ValidateConstant(JSContext* cx, const AsmJSGlobal& global, + HandleValue globalVal) { + RootedValue v(cx, globalVal); + + if (global.constantKind() == AsmJSGlobal::MathConstant) { + if (!GetDataProperty(cx, v, cx->names().Math, &v)) { + return false; + } + } + + if (!GetDataProperty(cx, v, global.field(), &v)) { + return false; + } + + if (!v.isNumber()) { + return LinkFail(cx, "math / global constant value needs to be a number"); + } + + // NaN != NaN + if (std::isnan(global.constantValue())) { + if (!std::isnan(v.toNumber())) { + return LinkFail(cx, "global constant value needs to be NaN"); + } + } else { + if (v.toNumber() != global.constantValue()) { + return LinkFail(cx, "global constant value mismatch"); + } + } + + return true; +} + +static bool CheckBuffer(JSContext* cx, const AsmJSMetadata& metadata, + HandleValue bufferVal, + MutableHandle buffer) { + if (!bufferVal.isObject()) { + return LinkFail(cx, "buffer must be an object"); + } + JSObject* bufferObj = &bufferVal.toObject(); + + if (metadata.usesSharedMemory()) { + if (!bufferObj->is()) { + return LinkFail( + cx, "shared views can only be constructed onto SharedArrayBuffer"); + } + return LinkFail(cx, "Unable to prepare SharedArrayBuffer for asm.js use"); + } + + if (!bufferObj->is()) { + return LinkFail(cx, + "unshared views can only be constructed onto ArrayBuffer"); + } + + buffer.set(&bufferObj->as()); + + size_t memoryLength = buffer->byteLength(); + + if (!IsValidAsmJSHeapLength(memoryLength)) { + UniqueChars msg; + if (memoryLength > MaxHeapLength) { + msg = JS_smprintf("ArrayBuffer byteLength 0x%" PRIx64 + " is not a valid heap length - it is too long." + " The longest valid length is 0x%" PRIx64, + uint64_t(memoryLength), MaxHeapLength); + } else { + msg = JS_smprintf("ArrayBuffer byteLength 0x%" PRIx64 + " is not a valid heap length. The next " + "valid length is 0x%" PRIx64, + uint64_t(memoryLength), + RoundUpToNextValidAsmJSHeapLength(memoryLength)); + } + if (!msg) { + return false; + } + return LinkFail(cx, msg.get()); + } + + // This check is sufficient without considering the size of the loaded datum + // because heap loads and stores start on an aligned boundary and the heap + // byteLength has larger alignment. + uint64_t minMemoryLength = + metadata.usesMemory() ? metadata.memory->initialLength32() : 0; + MOZ_ASSERT((minMemoryLength - 1) <= INT32_MAX); + if (memoryLength < minMemoryLength) { + UniqueChars msg(JS_smprintf("ArrayBuffer byteLength of 0x%" PRIx64 + " is less than 0x%" PRIx64 " (the " + "size implied " + "by const heap accesses).", + uint64_t(memoryLength), minMemoryLength)); + if (!msg) { + return false; + } + return LinkFail(cx, msg.get()); + } + + // ArrayBuffer lengths in SpiderMonkey used to be restricted to <= INT32_MAX, + // but that has since been relaxed for the benefit of wasm. We keep the old + // limit for asm.js so as to avoid having to worry about whether the asm.js + // implementation is safe for larger heaps. + if (memoryLength >= INT32_MAX) { + UniqueChars msg( + JS_smprintf("ArrayBuffer byteLength 0x%" PRIx64 + " is too large for asm.js (implementation limit).", + uint64_t(memoryLength))); + if (!msg) { + return false; + } + return LinkFail(cx, msg.get()); + } + + if (!buffer->prepareForAsmJS()) { + return LinkFail(cx, "Unable to prepare ArrayBuffer for asm.js use"); + } + + MOZ_ASSERT(buffer->isPreparedForAsmJS()); + return true; +} + +static bool GetImports(JSContext* cx, const AsmJSMetadata& metadata, + HandleValue globalVal, HandleValue importVal, + ImportValues* imports) { + Rooted ffis(cx, FunctionVector(cx)); + if (!ffis.resize(metadata.numFFIs)) { + return false; + } + + for (const AsmJSGlobal& global : metadata.asmJSGlobals) { + switch (global.which()) { + case AsmJSGlobal::Variable: { + Maybe litVal; + if (!ValidateGlobalVariable(cx, global, importVal, &litVal)) { + return false; + } + if (!imports->globalValues.append(Val(litVal->asLitVal()))) { + return false; + } + break; + } + case AsmJSGlobal::FFI: + if (!ValidateFFI(cx, global, importVal, &ffis)) { + return false; + } + break; + case AsmJSGlobal::ArrayView: + case AsmJSGlobal::ArrayViewCtor: + if (!ValidateArrayView(cx, global, globalVal)) { + return false; + } + break; + case AsmJSGlobal::MathBuiltinFunction: + if (!ValidateMathBuiltinFunction(cx, metadata, global, globalVal)) { + return false; + } + break; + case AsmJSGlobal::Constant: + if (!ValidateConstant(cx, global, globalVal)) { + return false; + } + break; + } + } + + for (const AsmJSImport& import : metadata.asmJSImports) { + if (!imports->funcs.append(ffis[import.ffiIndex()])) { + return false; + } + } + + return true; +} + +static bool TryInstantiate(JSContext* cx, CallArgs args, const Module& module, + const AsmJSMetadata& metadata, + MutableHandle instanceObj, + MutableHandleObject exportObj) { + HandleValue globalVal = args.get(0); + HandleValue importVal = args.get(1); + HandleValue bufferVal = args.get(2); + + // Re-check HasPlatformSupport(cx) since this varies per-thread and + // 'module' may have been produced on a parser thread. + if (!HasPlatformSupport(cx)) { + return LinkFail(cx, "no platform support"); + } + + Rooted imports(cx); + + if (module.metadata().usesMemory()) { + RootedArrayBufferObject buffer(cx); + if (!CheckBuffer(cx, metadata, bufferVal, &buffer)) { + return false; + } + + imports.get().memory = + WasmMemoryObject::create(cx, buffer, /* isHuge= */ false, nullptr); + if (!imports.get().memory) { + return false; + } + } + + if (!GetImports(cx, metadata, globalVal, importVal, imports.address())) { + return false; + } + + if (!module.instantiate(cx, imports.get(), nullptr, instanceObj)) { + return false; + } + + exportObj.set(&instanceObj->exportsObj()); + return true; +} + +static bool HandleInstantiationFailure(JSContext* cx, CallArgs args, + const AsmJSMetadata& metadata) { + using js::frontend::FunctionSyntaxKind; + + Rooted name(cx, args.callee().as().explicitName()); + + if (cx->isExceptionPending()) { + return false; + } + + ScriptSource* source = metadata.maybeScriptSource(); + + // Source discarding is allowed to affect JS semantics because it is never + // enabled for normal JS content. + bool haveSource; + if (!ScriptSource::loadSource(cx, source, &haveSource)) { + return false; + } + if (!haveSource) { + JS_ReportErrorASCII(cx, + "asm.js link failure with source discarding enabled"); + return false; + } + + uint32_t begin = metadata.toStringStart; + uint32_t end = metadata.srcEndAfterCurly(); + Rooted src(cx, source->substringDontDeflate(cx, begin, end)); + if (!src) { + return false; + } + + JS::CompileOptions options(cx); + options.setMutedErrors(source->mutedErrors()) + .setFile(source->filename()) + .setNoScriptRval(false); + options.asmJSOption = AsmJSOption::DisabledByLinker; + + // The exported function inherits an implicit strict context if the module + // also inherited it somehow. + if (metadata.strict) { + options.setForceStrictMode(); + } + + AutoStableStringChars linearChars(cx); + if (!linearChars.initTwoByte(cx, src)) { + return false; + } + + SourceText srcBuf; + if (!srcBuf.initMaybeBorrowed(cx, linearChars)) { + return false; + } + + FunctionSyntaxKind syntaxKind = FunctionSyntaxKind::Statement; + + RootedFunction fun(cx, frontend::CompileStandaloneFunction( + cx, options, srcBuf, Nothing(), syntaxKind)); + if (!fun) { + return false; + } + + fun->initEnvironment(&cx->global()->lexicalEnvironment()); + + // Call the function we just recompiled. + args.setCallee(ObjectValue(*fun)); + return InternalCallOrConstruct( + cx, args, args.isConstructing() ? CONSTRUCT : NO_CONSTRUCT); +} + +static const Module& AsmJSModuleFunctionToModule(JSFunction* fun) { + MOZ_ASSERT(IsAsmJSModule(fun)); + const Value& v = fun->getExtendedSlot(FunctionExtended::ASMJS_MODULE_SLOT); + return v.toObject().as().module(); +} + +// Implements the semantics of an asm.js module function that has been +// successfully validated. +bool js::InstantiateAsmJS(JSContext* cx, unsigned argc, JS::Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + JSFunction* callee = &args.callee().as(); + const Module& module = AsmJSModuleFunctionToModule(callee); + const AsmJSMetadata& metadata = module.metadata().asAsmJS(); + + Rooted instanceObj(cx); + RootedObject exportObj(cx); + if (!TryInstantiate(cx, args, module, metadata, &instanceObj, &exportObj)) { + // Link-time validation checks failed, so reparse the entire asm.js + // module from scratch to get normal interpreted bytecode which we can + // simply Invoke. Very slow. + return HandleInstantiationFailure(cx, args, metadata); + } + + args.rval().set(ObjectValue(*exportObj)); + return true; +} + +/*****************************************************************************/ +// Top-level js::CompileAsmJS + +static bool NoExceptionPending(FrontendContext* fc) { return !fc->hadErrors(); } + +static bool SuccessfulValidation(frontend::ParserBase& parser, + unsigned compilationTime) { + unsigned errNum = js::SupportDifferentialTesting() + ? JSMSG_USE_ASM_TYPE_OK_NO_TIME + : JSMSG_USE_ASM_TYPE_OK; + + char timeChars[20]; + SprintfLiteral(timeChars, "%u", compilationTime); + + return parser.warningNoOffset(errNum, timeChars); +} + +static bool TypeFailureWarning(frontend::ParserBase& parser, const char* str) { + if (parser.options().throwOnAsmJSValidationFailureOption) { + parser.errorNoOffset(JSMSG_USE_ASM_TYPE_FAIL, str ? str : ""); + return false; + } + + // Per the asm.js standard convention, whether failure sets a pending + // exception determines whether to attempt non-asm.js reparsing, so ignore + // the return value below. + (void)parser.warningNoOffset(JSMSG_USE_ASM_TYPE_FAIL, str ? str : ""); + return false; +} + +// asm.js requires Ion to be available on the current hardware/OS and to be +// enabled for wasm, since asm.js compilation goes via wasm. +static bool IsAsmJSCompilerAvailable(JSContext* cx) { + return HasPlatformSupport(cx) && WasmCompilerForAsmJSAvailable(cx); +} + +static bool EstablishPreconditions(frontend::ParserBase& parser) { + switch (parser.options().asmJSOption) { + case AsmJSOption::DisabledByAsmJSPref: + return TypeFailureWarning( + parser, "Asm.js optimizer disabled by 'asmjs' runtime option"); + case AsmJSOption::DisabledByLinker: + return TypeFailureWarning( + parser, + "Asm.js optimizer disabled by linker (instantiation failure)"); + case AsmJSOption::DisabledByNoWasmCompiler: + return TypeFailureWarning(parser, + "Asm.js optimizer disabled because no suitable " + "wasm compiler is available"); + case AsmJSOption::DisabledByDebugger: + return TypeFailureWarning( + parser, "Asm.js optimizer disabled because debugger is active"); + case AsmJSOption::Enabled: + break; + } + + if (parser.pc_->isGenerator()) { + return TypeFailureWarning(parser, + "Asm.js optimizer disabled in generator context"); + } + + if (parser.pc_->isAsync()) { + return TypeFailureWarning(parser, + "Asm.js optimizer disabled in async context"); + } + + if (parser.pc_->isArrowFunction()) { + return TypeFailureWarning( + parser, "Asm.js optimizer disabled in arrow function context"); + } + + // Class constructors are also methods + if (parser.pc_->isMethod() || parser.pc_->isGetterOrSetter()) { + return TypeFailureWarning( + parser, + "Asm.js optimizer disabled in class constructor or method context"); + } + + return true; +} + +template +static bool DoCompileAsmJS(FrontendContext* fc, ParserAtomsTable& parserAtoms, + AsmJSParser& parser, ParseNode* stmtList, + bool* validated) { + *validated = false; + + // Various conditions disable asm.js optimizations. + if (!EstablishPreconditions(parser)) { + return NoExceptionPending(fc); + } + + // "Checking" parses, validates and compiles, producing a fully compiled + // WasmModuleObject as result. + unsigned time; + SharedModule module = CheckModule(fc, parserAtoms, parser, stmtList, &time); + if (!module) { + return NoExceptionPending(fc); + } + + // Finished! Save the ref-counted module on the FunctionBox. When JSFunctions + // are eventually allocated we will create an asm.js constructor for it. + FunctionBox* funbox = parser.pc_->functionBox(); + MOZ_ASSERT(funbox->isInterpreted()); + if (!funbox->setAsmJSModule(module)) { + return NoExceptionPending(fc); + } + + // Success! Write to the console with a "warning" message indicating + // total compilation time. + *validated = true; + SuccessfulValidation(parser, time); + return NoExceptionPending(fc); +} + +bool js::CompileAsmJS(FrontendContext* fc, ParserAtomsTable& parserAtoms, + AsmJSParser& parser, ParseNode* stmtList, + bool* validated) { + return DoCompileAsmJS(fc, parserAtoms, parser, stmtList, validated); +} + +bool js::CompileAsmJS(FrontendContext* fc, ParserAtomsTable& parserAtoms, + AsmJSParser& parser, ParseNode* stmtList, + bool* validated) { + return DoCompileAsmJS(fc, parserAtoms, parser, stmtList, validated); +} + +/*****************************************************************************/ +// asm.js testing functions + +bool js::IsAsmJSModuleNative(Native native) { + return native == InstantiateAsmJS; +} + +bool js::IsAsmJSModule(JSFunction* fun) { + return fun->maybeNative() == InstantiateAsmJS; +} + +bool js::IsAsmJSFunction(JSFunction* fun) { + return fun->kind() == FunctionFlags::AsmJS; +} + +bool js::IsAsmJSStrictModeModuleOrFunction(JSFunction* fun) { + if (IsAsmJSModule(fun)) { + return AsmJSModuleFunctionToModule(fun).metadata().asAsmJS().strict; + } + + if (IsAsmJSFunction(fun)) { + return ExportedFunctionToInstance(fun).metadata().asAsmJS().strict; + } + + return false; +} + +bool js::IsAsmJSCompilationAvailable(JSContext* cx) { + return cx->options().asmJS() && IsAsmJSCompilerAvailable(cx); +} + +bool js::IsAsmJSCompilationAvailable(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + bool available = IsAsmJSCompilationAvailable(cx); + args.rval().set(BooleanValue(available)); + return true; +} + +static JSFunction* MaybeWrappedNativeFunction(const Value& v) { + if (!v.isObject()) { + return nullptr; + } + + return v.toObject().maybeUnwrapIf(); +} + +bool js::IsAsmJSModule(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + bool rval = false; + if (JSFunction* fun = MaybeWrappedNativeFunction(args.get(0))) { + rval = IsAsmJSModule(fun); + } + + args.rval().set(BooleanValue(rval)); + return true; +} + +bool js::IsAsmJSFunction(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + bool rval = false; + if (JSFunction* fun = MaybeWrappedNativeFunction(args.get(0))) { + rval = IsAsmJSFunction(fun); + } + + args.rval().set(BooleanValue(rval)); + return true; +} + +/*****************************************************************************/ +// asm.js toString/toSource support + +JSString* js::AsmJSModuleToString(JSContext* cx, HandleFunction fun, + bool isToSource) { + MOZ_ASSERT(IsAsmJSModule(fun)); + + const AsmJSMetadata& metadata = + AsmJSModuleFunctionToModule(fun).metadata().asAsmJS(); + uint32_t begin = metadata.toStringStart; + uint32_t end = metadata.srcEndAfterCurly(); + ScriptSource* source = metadata.maybeScriptSource(); + + JSStringBuilder out(cx); + + if (isToSource && fun->isLambda() && !out.append("(")) { + return nullptr; + } + + bool haveSource; + if (!ScriptSource::loadSource(cx, source, &haveSource)) { + return nullptr; + } + + if (!haveSource) { + if (!out.append("function ")) { + return nullptr; + } + if (fun->explicitName() && !out.append(fun->explicitName())) { + return nullptr; + } + if (!out.append("() {\n [native code]\n}")) { + return nullptr; + } + } else { + Rooted src(cx, source->substring(cx, begin, end)); + if (!src) { + return nullptr; + } + + if (!out.append(src)) { + return nullptr; + } + } + + if (isToSource && fun->isLambda() && !out.append(")")) { + return nullptr; + } + + return out.finishString(); +} + +JSString* js::AsmJSFunctionToString(JSContext* cx, HandleFunction fun) { + MOZ_ASSERT(IsAsmJSFunction(fun)); + + const AsmJSMetadata& metadata = + ExportedFunctionToInstance(fun).metadata().asAsmJS(); + const AsmJSExport& f = + metadata.lookupAsmJSExport(ExportedFunctionToFuncIndex(fun)); + + uint32_t begin = metadata.srcStart + f.startOffsetInModule(); + uint32_t end = metadata.srcStart + f.endOffsetInModule(); + + ScriptSource* source = metadata.maybeScriptSource(); + JSStringBuilder out(cx); + + if (!out.append("function ")) { + return nullptr; + } + + bool haveSource; + if (!ScriptSource::loadSource(cx, source, &haveSource)) { + return nullptr; + } + + if (!haveSource) { + // asm.js functions can't be anonymous + MOZ_ASSERT(fun->explicitName()); + if (!out.append(fun->explicitName())) { + return nullptr; + } + if (!out.append("() {\n [native code]\n}")) { + return nullptr; + } + } else { + Rooted src(cx, source->substring(cx, begin, end)); + if (!src) { + return nullptr; + } + if (!out.append(src)) { + return nullptr; + } + } + + return out.finishString(); +} + +bool js::IsValidAsmJSHeapLength(size_t length) { + if (length < MinHeapLength) { + return false; + } + + // The heap length is limited by what a wasm memory32 can handle. + if (length > MaxMemoryBytes(IndexType::I32)) { + return false; + } + + return wasm::IsValidARMImmediate(length); +} diff --git a/js/src/wasm/AsmJS.h b/js/src/wasm/AsmJS.h new file mode 100644 index 0000000000..7caa1bf09a --- /dev/null +++ b/js/src/wasm/AsmJS.h @@ -0,0 +1,116 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2014 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_AsmJS_h +#define wasm_AsmJS_h + +#include "mozilla/Utf8.h" // mozilla::Utf8Unit + +#include // uint32_t + +#include "jstypes.h" // JS_PUBLIC_API +#include "js/CallArgs.h" // JSNative + +struct JS_PUBLIC_API JSContext; +class JS_PUBLIC_API JSFunction; + +namespace JS { + +class JS_PUBLIC_API Value; + +template +class Handle; + +} // namespace JS + +namespace js { + +class FrontendContext; + +namespace frontend { + +class ParserAtomsTable; +class ParseContext; +class ParseNode; + +template +class Parser; +class FullParseHandler; + +} // namespace frontend + +template +using AsmJSParser = frontend::Parser; + +// This function takes over parsing of a function starting with "use asm". The +// return value indicates whether an error was reported which the caller should +// propagate. If no error was reported, the function may still fail to validate +// as asm.js. In this case, the parser.tokenStream has been advanced an +// indeterminate amount and the entire function should be reparsed from the +// beginning. + +[[nodiscard]] extern bool CompileAsmJS(FrontendContext* fc, + frontend::ParserAtomsTable& parserAtoms, + AsmJSParser& parser, + frontend::ParseNode* stmtList, + bool* validated); + +[[nodiscard]] extern bool CompileAsmJS(FrontendContext* fc, + frontend::ParserAtomsTable& parserAtoms, + AsmJSParser& parser, + frontend::ParseNode* stmtList, + bool* validated); + +// asm.js module/export queries: + +extern bool IsAsmJSModuleNative(JSNative native); + +extern bool IsAsmJSModule(JSFunction* fun); + +extern bool IsAsmJSFunction(JSFunction* fun); + +extern bool IsAsmJSStrictModeModuleOrFunction(JSFunction* fun); + +extern bool InstantiateAsmJS(JSContext* cx, unsigned argc, JS::Value* vp); + +// asm.js testing natives: + +extern bool IsAsmJSCompilationAvailable(JSContext* cx, unsigned argc, + JS::Value* vp); + +extern bool IsAsmJSCompilationAvailable(JSContext* cx); + +extern bool IsAsmJSModule(JSContext* cx, unsigned argc, JS::Value* vp); + +extern bool IsAsmJSFunction(JSContext* cx, unsigned argc, JS::Value* vp); + +// asm.js toString/toSource support: + +extern JSString* AsmJSFunctionToString(JSContext* cx, + JS::Handle fun); + +extern JSString* AsmJSModuleToString(JSContext* cx, JS::Handle fun, + bool isToSource); + +// asm.js heap: + +extern bool IsValidAsmJSHeapLength(size_t length); + +} // namespace js + +#endif // wasm_AsmJS_h diff --git a/js/src/wasm/GenerateIntrinsics.py b/js/src/wasm/GenerateIntrinsics.py new file mode 100644 index 0000000000..f822722dcd --- /dev/null +++ b/js/src/wasm/GenerateIntrinsics.py @@ -0,0 +1,86 @@ +from collections import OrderedDict + +import buildconfig +import six +import yaml +from mozbuild.preprocessor import Preprocessor + +HEADER_TEMPLATE = """\ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef %(includeguard)s +#define %(includeguard)s + +/* This file is generated by wasm/GenerateInstrinsic.py. Do not edit! */ + +%(contents)s + +#endif // %(includeguard)s +""" + + +def generate_header(c_out, includeguard, contents): + c_out.write( + HEADER_TEMPLATE + % { + "includeguard": includeguard, + "contents": contents, + } + ) + + +def load_yaml(yaml_path): + # First invoke preprocessor.py so that we can use #ifdef JS_SIMULATOR in + # the YAML file. + pp = Preprocessor() + pp.context.update(buildconfig.defines["ALLDEFINES"]) + pp.out = six.StringIO() + pp.do_filter("substitution") + pp.do_include(yaml_path) + contents = pp.out.getvalue() + + # Load into an OrderedDict to ensure order is preserved. Note: Python 3.7+ + # also preserves ordering for normal dictionaries. + # Code based on https://stackoverflow.com/a/21912744. + class OrderedLoader(yaml.Loader): + pass + + def construct_mapping(loader, node): + loader.flatten_mapping(node) + return OrderedDict(loader.construct_pairs(node)) + + tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG + OrderedLoader.add_constructor(tag, construct_mapping) + return yaml.load(contents, OrderedLoader) + + +def main(c_out, yaml_path): + data = load_yaml(yaml_path) + + # Interate for all defined intrinsics + contents = "#define FOR_EACH_INTRINSIC(M) \\\n" + for i in range(len(data)): + op = data[i] + sa = op["symbolic_address"] + contents += ( + f" M({op['op']}, \"{op['export']}\", " + f"{sa['name']}, {sa['type']}, {op['entry']}, {i})\\\n" + ) + contents += "\n" + + for op in data: + # Define DECLARE_INTRINSIC_SAS_PARAM_VALTYPES_ as: + # `{ValType::I32, ValType::I32, ...}`. + contents += ( + f"#define DECLARE_INTRINSIC_SAS_PARAM_VALTYPES_{op['op']} " + f"{{ValType::{', ValType::'.join(op['params'])}}}\n" + ) + # Define DECLARE_INTRINSIC_PARAM_TYPES_ as: + # `, {_PTR, _I32, ..., _PTR, _END}`. + sas_types = f"{{_PTR{''.join(', _' + p for p in op['params'])}, _PTR, _END}}" + num_types = len(op["params"]) + 2 + contents += f"#define DECLARE_INTRINSIC_PARAM_TYPES_{op['op']} {num_types}, {sas_types}\n" + + generate_header(c_out, "wasm_WasmIntrinsicGenerated_h", contents) diff --git a/js/src/wasm/WasmBCClass-inl.h b/js/src/wasm/WasmBCClass-inl.h new file mode 100644 index 0000000000..56d174506e --- /dev/null +++ b/js/src/wasm/WasmBCClass-inl.h @@ -0,0 +1,58 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: inline BaseCompiler +// methods that don't fit in any other group in particular. + +#ifndef wasm_wasm_baseline_object_inl_h +#define wasm_wasm_baseline_object_inl_h + +namespace js { +namespace wasm { + +const FuncType& BaseCompiler::funcType() const { + return *moduleEnv_.funcs[func_.index].type; +} + +bool BaseCompiler::usesMemory() const { return moduleEnv_.usesMemory(); } + +bool BaseCompiler::usesSharedMemory() const { + return moduleEnv_.usesSharedMemory(); +} + +const Local& BaseCompiler::localFromSlot(uint32_t slot, MIRType type) { + MOZ_ASSERT(localInfo_[slot].type == type); + return localInfo_[slot]; +} + +BytecodeOffset BaseCompiler::bytecodeOffset() const { + return iter_.bytecodeOffset(); +} + +bool BaseCompiler::isMem32() const { + return moduleEnv_.memory->indexType() == IndexType::I32; +} + +bool BaseCompiler::isMem64() const { + return moduleEnv_.memory->indexType() == IndexType::I64; +} + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_object_inl_h diff --git a/js/src/wasm/WasmBCClass.h b/js/src/wasm/WasmBCClass.h new file mode 100644 index 0000000000..13b5ff094c --- /dev/null +++ b/js/src/wasm/WasmBCClass.h @@ -0,0 +1,1745 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: the compiler object +// and its supporting types. + +#ifndef wasm_wasm_baseline_object_h +#define wasm_wasm_baseline_object_h + +#include "wasm/WasmBCDefs.h" +#include "wasm/WasmBCFrame.h" +#include "wasm/WasmBCRegDefs.h" +#include "wasm/WasmBCStk.h" + +namespace js { +namespace wasm { + +// Container for a piece of out-of-line code, the slow path that supports an +// operation. +class OutOfLineCode; + +// Part of the inter-bytecode state for the boolean-evaluation-for-control +// optimization. +struct BranchState; + +// Representation of wasm local variables. +using Local = BaseStackFrame::Local; + +// Bitset used for simple bounds check elimination. Capping this at 64 locals +// makes sense; even 32 locals would probably be OK in practice. +// +// For more information about BCE, see the block comment in WasmBCMemory.cpp. +using BCESet = uint64_t; + +// Information stored in the control node for generating exception handling +// landing pads. +struct CatchInfo { + uint32_t tagIndex; // Index for the associated exception. + NonAssertingLabel label; // The entry label for the handler. + + explicit CatchInfo(uint32_t tagIndex_) : tagIndex(tagIndex_) {} +}; + +using CatchInfoVector = Vector; + +// Control node, representing labels and stack heights at join points. +struct Control { + NonAssertingLabel label; // The "exit" label + NonAssertingLabel otherLabel; // Used for the "else" branch of if-then-else + // and to allow delegate to jump to catches. + StackHeight stackHeight; // From BaseStackFrame + uint32_t stackSize; // Value stack height + BCESet bceSafeOnEntry; // Bounds check info flowing into the item + BCESet bceSafeOnExit; // Bounds check info flowing out of the item + bool deadOnArrival; // deadCode_ was set on entry to the region + bool deadThenBranch; // deadCode_ was set on exit from "then" + size_t tryNoteIndex; // For tracking try branch code ranges. + CatchInfoVector catchInfos; // Used for try-catch handlers. + + Control() + : stackHeight(StackHeight::Invalid()), + stackSize(UINT32_MAX), + bceSafeOnEntry(0), + bceSafeOnExit(~BCESet(0)), + deadOnArrival(false), + deadThenBranch(false), + tryNoteIndex(0) {} +}; + +// A vector of Nothing values, used for reading opcodes. +class BaseNothingVector { + Nothing unused_; + + public: + bool resize(size_t length) { return true; } + Nothing& operator[](size_t) { return unused_; } + Nothing& back() { return unused_; } + size_t length() const { return 0; } + bool append(Nothing& nothing) { return true; } +}; + +// The baseline compiler tracks values on a stack of its own -- it needs to scan +// that stack for spilling -- and thus has no need for the values maintained by +// the iterator. +struct BaseCompilePolicy { + using Value = Nothing; + using ValueVector = BaseNothingVector; + + // The baseline compiler uses the iterator's control stack, attaching + // its own control information. + using ControlItem = Control; +}; + +using BaseOpIter = OpIter; + +// Latent operation for boolean-evaluation-for-control optimization. +enum class LatentOp { None, Compare, Eqz }; + +// Encapsulate the checking needed for a memory access. +struct AccessCheck { + AccessCheck() + : omitBoundsCheck(false), + omitAlignmentCheck(false), + onlyPointerAlignment(false) {} + + // If `omitAlignmentCheck` is true then we need check neither the + // pointer nor the offset. Otherwise, if `onlyPointerAlignment` is true + // then we need check only the pointer. Otherwise, check the sum of + // pointer and offset. + + bool omitBoundsCheck; + bool omitAlignmentCheck; + bool onlyPointerAlignment; +}; + +// Encapsulate all the information about a function call. +struct FunctionCall { + FunctionCall() + : restoreRegisterStateAndRealm(false), + usesSystemAbi(false), +#ifdef JS_CODEGEN_ARM + hardFP(true), +#endif + frameAlignAdjustment(0), + stackArgAreaSize(0) { + } + + WasmABIArgGenerator abi; + bool restoreRegisterStateAndRealm; + bool usesSystemAbi; +#ifdef JS_CODEGEN_ARM + bool hardFP; +#endif + size_t frameAlignAdjustment; + size_t stackArgAreaSize; +}; + +enum class PreBarrierKind { + // No pre-write barrier is required because the previous value is undefined. + None, + // Perform a pre-write barrier to mark the previous value if an incremental + // GC is underway. + Normal, +}; + +enum class PostBarrierKind { + // Remove an existing store buffer entry if the new value does not require + // one. This is required to preserve invariants with HeapPtr when used for + // movable storage. + Precise, + // Add a store buffer entry if the new value requires it, but do not attempt + // to remove a pre-existing entry. + Imprecise, +}; + +////////////////////////////////////////////////////////////////////////////// +// +// Wasm baseline compiler proper. +// +// This is a struct and not a class because there is no real benefit to hiding +// anything, and because many static functions that are wrappers for masm +// methods need to reach into it and would otherwise have to be declared as +// friends. +// +// (Members generally have a '_' suffix but some don't because they are +// referenced everywhere and it would be tedious to spell that out.) + +struct BaseCompiler final { + /////////////////////////////////////////////////////////////////////////// + // + // Private types + + using LabelVector = Vector; + + /////////////////////////////////////////////////////////////////////////// + // + // Read-only and write-once members. + + // Static compilation environment. + const ModuleEnvironment& moduleEnv_; + const CompilerEnvironment& compilerEnv_; + const FuncCompileInput& func_; + const ValTypeVector& locals_; + + // Information about the locations of locals, this is set up during + // initialization and read-only after that. + BaseStackFrame::LocalVector localInfo_; + + // On specific platforms we sometimes need to use specific registers. + const SpecificRegs specific_; + + // SigD and SigF are single-entry parameter lists for f64 and f32, these are + // created during initialization. + ValTypeVector SigD_; + ValTypeVector SigF_; + + // Where to go to to return, bound as compilation ends. + NonAssertingLabel returnLabel_; + + // Prologue and epilogue offsets, initialized during prologue and epilogue + // generation and only used by the caller. + FuncOffsets offsets_; + + // We call this address from the breakable point when the breakpoint handler + // is not null. + NonAssertingLabel debugTrapStub_; + uint32_t previousBreakablePoint_; + + // BaselineCompileFunctions() "lends" us the StkVector to use in this + // BaseCompiler object, and that is installed in |stk_| in our constructor. + // This is so as to avoid having to malloc/free the vector's contents at + // each creation/destruction of a BaseCompiler object. It does however mean + // that we need to hold on to a reference to BaselineCompileFunctions()'s + // vector, so we can swap (give) its contents back when this BaseCompiler + // object is destroyed. This significantly reduces the heap turnover of the + // baseline compiler. See bug 1532592. + StkVector& stkSource_; + + /////////////////////////////////////////////////////////////////////////// + // + // Output-only data structures. + + // Bump allocator for temporary memory, used for the value stack and + // out-of-line code blobs. Bump-allocated memory is not freed until the end + // of the compilation. + TempAllocator::Fallible alloc_; + + // Machine code emitter. + MacroAssembler& masm; + + /////////////////////////////////////////////////////////////////////////// + // + // Compilation state. + + // Decoder for this function, used for misc error reporting. + Decoder& decoder_; + + // Opcode reader. + BaseOpIter iter_; + + // Register allocator. + BaseRegAlloc ra; + + // Stack frame abstraction. + BaseStackFrame fr; + + // Latent out of line support code for some operations, code for these will be + // emitted at the end of compilation. + Vector outOfLine_; + + // Stack map state. This keeps track of live pointer slots and allows precise + // stack maps to be generated at safe points. + StackMapGenerator stackMapGenerator_; + + // Wasm value stack. This maps values on the wasm stack to values in the + // running code and their locations. + // + // The value stack facilitates on-the-fly register allocation and the use of + // immediates in instructions. It tracks latent constants, latent references + // to locals, register contents, and values that have been flushed to the CPU + // stack. + // + // The stack can be flushed to the CPU stack using sync(). + // + // The stack is a StkVector rather than a StkVector& since constantly + // dereferencing a StkVector& has been shown to add 0.5% or more to the + // compiler's dynamic instruction count. + StkVector stk_; + + // Flag indicating that the compiler is currently in a dead code region. + bool deadCode_; + + /////////////////////////////////////////////////////////////////////////// + // + // State for bounds check elimination. + + // Locals that have been bounds checked and not updated since + BCESet bceSafe_; + + /////////////////////////////////////////////////////////////////////////// + // + // State for boolean-evaluation-for-control. + + // Latent operation for branch (seen next) + LatentOp latentOp_; + + // Operand type, if latentOp_ is true + ValType latentType_; + + // Comparison operator, if latentOp_ == Compare, int types + Assembler::Condition latentIntCmp_; + + // Comparison operator, if latentOp_ == Compare, float types + Assembler::DoubleCondition latentDoubleCmp_; + + /////////////////////////////////////////////////////////////////////////// + // + // Main compilation API. + // + // A client will create a compiler object, and then call init(), + // emitFunction(), and finish() in that order. + + BaseCompiler(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + const FuncCompileInput& func, const ValTypeVector& locals, + const RegisterOffsets& trapExitLayout, + size_t trapExitLayoutNumWords, Decoder& decoder, + StkVector& stkSource, TempAllocator* alloc, MacroAssembler* masm, + StackMaps* stackMaps); + ~BaseCompiler(); + + [[nodiscard]] bool init(); + [[nodiscard]] bool emitFunction(); + [[nodiscard]] FuncOffsets finish(); + + ////////////////////////////////////////////////////////////////////////////// + // + // Sundry accessor abstractions and convenience predicates. + // + // WasmBaselineObject-inl.h. + + inline const FuncType& funcType() const; + inline bool usesMemory() const; + inline bool usesSharedMemory() const; + inline bool isMem32() const; + inline bool isMem64() const; + + // The casts are used by some of the ScratchRegister implementations. + operator MacroAssembler&() const { return masm; } + operator BaseRegAlloc&() { return ra; } + + ////////////////////////////////////////////////////////////////////////////// + // + // Locals. + // + // WasmBaselineObject-inl.h. + + // Assert that the local at the given index has the given type, and return a + // reference to the Local. + inline const Local& localFromSlot(uint32_t slot, MIRType type); + + ////////////////////////////////////////////////////////////////////////////// + // + // Out of line code management. + + [[nodiscard]] OutOfLineCode* addOutOfLineCode(OutOfLineCode* ool); + [[nodiscard]] bool generateOutOfLineCode(); + + ///////////////////////////////////////////////////////////////////////////// + // + // Layering in the compiler (briefly). + // + // At the lowest layers are abstractions for registers (managed by the + // BaseRegAlloc and the wrappers below) and the stack frame (managed by the + // BaseStackFrame). + // + // The registers and frame are in turn used by the value abstraction, which is + // implemented by the Stk type and backed by the value stack. Values may be + // stored in registers, in the frame, or may be latent constants, and the + // value stack handles storage mostly transparently in its push and pop + // routines. + // + // In turn, the pop routines bring values into registers so that we can + // compute on them, and the push routines move values to the stack (where they + // may still reside in registers until the registers are needed or the value + // must be in memory). + // + // Routines for managing parameters and results (for blocks or calls) may also + // manipulate the stack directly. + // + // At the top are the code generators: methods that use the poppers and + // pushers and other utilities to move values into place, and that emit code + // to compute on those values or change control flow. + + ///////////////////////////////////////////////////////////////////////////// + // + // Register management. These are simply strongly-typed wrappers that + // delegate to the register allocator. + + inline bool isAvailableI32(RegI32 r); + inline bool isAvailableI64(RegI64 r); + inline bool isAvailableRef(RegRef r); + inline bool isAvailablePtr(RegPtr r); + inline bool isAvailableF32(RegF32 r); + inline bool isAvailableF64(RegF64 r); +#ifdef ENABLE_WASM_SIMD + inline bool isAvailableV128(RegV128 r); +#endif + + // Allocate any register + [[nodiscard]] inline RegI32 needI32(); + [[nodiscard]] inline RegI64 needI64(); + [[nodiscard]] inline RegRef needRef(); + [[nodiscard]] inline RegPtr needPtr(); + [[nodiscard]] inline RegF32 needF32(); + [[nodiscard]] inline RegF64 needF64(); +#ifdef ENABLE_WASM_SIMD + [[nodiscard]] inline RegV128 needV128(); +#endif + + // Allocate a specific register + inline void needI32(RegI32 specific); + inline void needI64(RegI64 specific); + inline void needRef(RegRef specific); + inline void needPtr(RegPtr specific); + inline void needF32(RegF32 specific); + inline void needF64(RegF64 specific); +#ifdef ENABLE_WASM_SIMD + inline void needV128(RegV128 specific); +#endif + + template + inline RegType need(); + + // Just a shorthand. + inline void need2xI32(RegI32 r0, RegI32 r1); + inline void need2xI64(RegI64 r0, RegI64 r1); + + // Get a register but do not sync the stack to free one up. This will crash + // if no register is available. + inline void needI32NoSync(RegI32 r); + +#if defined(JS_CODEGEN_ARM) + // Allocate a specific register pair (even-odd register numbers). + [[nodiscard]] inline RegI64 needI64Pair(); +#endif + + inline void freeAny(AnyReg r); + inline void freeI32(RegI32 r); + inline void freeI64(RegI64 r); + inline void freeRef(RegRef r); + inline void freePtr(RegPtr r); + inline void freeF32(RegF32 r); + inline void freeF64(RegF64 r); +#ifdef ENABLE_WASM_SIMD + inline void freeV128(RegV128 r); +#endif + + template + inline void free(RegType r); + + // Free r if it is not invalid. + inline void maybeFree(RegI32 r); + inline void maybeFree(RegI64 r); + inline void maybeFree(RegF32 r); + inline void maybeFree(RegF64 r); + inline void maybeFree(RegRef r); + inline void maybeFree(RegPtr r); +#ifdef ENABLE_WASM_SIMD + inline void maybeFree(RegV128 r); +#endif + + // On 64-bit systems, `except` must equal r and this is a no-op. On 32-bit + // systems, `except` must equal the high or low part of a pair and the other + // part of the pair is freed. + inline void freeI64Except(RegI64 r, RegI32 except); + + // Return the 32-bit low part of the 64-bit register, do not free anything. + inline RegI32 fromI64(RegI64 r); + + // If r is valid, return fromI64(r), otherwise an invalid RegI32. + inline RegI32 maybeFromI64(RegI64 r); + +#ifdef JS_PUNBOX64 + // On 64-bit systems, reinterpret r as 64-bit. + inline RegI64 fromI32(RegI32 r); +#endif + + // Widen r to 64 bits; this may allocate another register to form a pair. + // Note this does not generate code for sign/zero extension. + inline RegI64 widenI32(RegI32 r); + + // Narrow r to 32 bits; this may free part of a pair. Note this does not + // generate code to canonicalize the value on 64-bit systems. + inline RegI32 narrowI64(RegI64 r); + inline RegI32 narrowRef(RegRef r); + + // Return the 32-bit low part of r. + inline RegI32 lowPart(RegI64 r); + + // On 64-bit systems, return an invalid register. On 32-bit systems, return + // the low part of a pair. + inline RegI32 maybeHighPart(RegI64 r); + + // On 64-bit systems, do nothing. On 32-bit systems, clear the high register. + inline void maybeClearHighPart(RegI64 r); + + ////////////////////////////////////////////////////////////////////////////// + // + // Values and value stack: Low-level methods for moving Stk values of specific + // kinds to registers. + + inline void loadConstI32(const Stk& src, RegI32 dest); + inline void loadMemI32(const Stk& src, RegI32 dest); + inline void loadLocalI32(const Stk& src, RegI32 dest); + inline void loadRegisterI32(const Stk& src, RegI32 dest); + inline void loadConstI64(const Stk& src, RegI64 dest); + inline void loadMemI64(const Stk& src, RegI64 dest); + inline void loadLocalI64(const Stk& src, RegI64 dest); + inline void loadRegisterI64(const Stk& src, RegI64 dest); + inline void loadConstRef(const Stk& src, RegRef dest); + inline void loadMemRef(const Stk& src, RegRef dest); + inline void loadLocalRef(const Stk& src, RegRef dest); + inline void loadRegisterRef(const Stk& src, RegRef dest); + inline void loadConstF64(const Stk& src, RegF64 dest); + inline void loadMemF64(const Stk& src, RegF64 dest); + inline void loadLocalF64(const Stk& src, RegF64 dest); + inline void loadRegisterF64(const Stk& src, RegF64 dest); + inline void loadConstF32(const Stk& src, RegF32 dest); + inline void loadMemF32(const Stk& src, RegF32 dest); + inline void loadLocalF32(const Stk& src, RegF32 dest); + inline void loadRegisterF32(const Stk& src, RegF32 dest); +#ifdef ENABLE_WASM_SIMD + inline void loadConstV128(const Stk& src, RegV128 dest); + inline void loadMemV128(const Stk& src, RegV128 dest); + inline void loadLocalV128(const Stk& src, RegV128 dest); + inline void loadRegisterV128(const Stk& src, RegV128 dest); +#endif + + ////////////////////////////////////////////////////////////////////////// + // + // Values and value stack: Mid-level routines for moving Stk values of any + // kind to registers. + + inline void loadI32(const Stk& src, RegI32 dest); + inline void loadI64(const Stk& src, RegI64 dest); +#if !defined(JS_PUNBOX64) + inline void loadI64Low(const Stk& src, RegI32 dest); + inline void loadI64High(const Stk& src, RegI32 dest); +#endif + inline void loadF64(const Stk& src, RegF64 dest); + inline void loadF32(const Stk& src, RegF32 dest); +#ifdef ENABLE_WASM_SIMD + inline void loadV128(const Stk& src, RegV128 dest); +#endif + inline void loadRef(const Stk& src, RegRef dest); + + ////////////////////////////////////////////////////////////////////// + // + // Value stack: stack management. + + // Flush all local and register value stack elements to memory. + inline void sync(); + + // Save a register on the value stack temporarily. + void saveTempPtr(const RegPtr& r); + + // Restore a temporarily saved register from the value stack. + void restoreTempPtr(const RegPtr& r); + + // This is an optimization used to avoid calling sync for setLocal: if the + // local does not exist unresolved on the value stack then we can skip the + // sync. + inline bool hasLocal(uint32_t slot); + + // Sync the local if necessary. (This currently syncs everything if a sync is + // needed at all.) + inline void syncLocal(uint32_t slot); + + // Return the amount of execution stack consumed by the top numval + // values on the value stack. + inline size_t stackConsumed(size_t numval); + + // Drop one value off the stack, possibly also moving the physical stack + // pointer. + inline void dropValue(); + +#ifdef DEBUG + // Check that we're not leaking registers by comparing the + // state of the stack + available registers with the set of + // all available registers. + + // Call this between opcodes. + void performRegisterLeakCheck(); + + // This can be called at any point, really, but typically just after + // performRegisterLeakCheck(). + void assertStackInvariants() const; + + // Count the number of memory references on the value stack. + inline size_t countMemRefsOnStk(); + + // Print the stack to stderr. + void showStack(const char* who) const; +#endif + + ////////////////////////////////////////////////////////////////////// + // + // Value stack: pushers of values. + + // Push a register onto the value stack. + inline void pushAny(AnyReg r); + inline void pushI32(RegI32 r); + inline void pushI64(RegI64 r); + inline void pushRef(RegRef r); + inline void pushPtr(RegPtr r); + inline void pushF64(RegF64 r); + inline void pushF32(RegF32 r); +#ifdef ENABLE_WASM_SIMD + inline void pushV128(RegV128 r); +#endif + + // Template variation of the foregoing, for use by templated emitters. + template + inline void push(RegType item); + + // Push a constant value onto the stack. pushI32 can also take uint32_t, and + // pushI64 can take uint64_t; the semantics are the same. Appropriate sign + // extension for a 32-bit value on a 64-bit architecture happens when the + // value is popped, see the definition of moveImm32. + inline void pushI32(int32_t v); + inline void pushI64(int64_t v); + inline void pushRef(intptr_t v); + inline void pushPtr(intptr_t v); + inline void pushF64(double v); + inline void pushF32(float v); +#ifdef ENABLE_WASM_SIMD + inline void pushV128(V128 v); +#endif + inline void pushConstRef(intptr_t v); + + // Push the local slot onto the stack. The slot will not be read here; it + // will be read when it is consumed, or when a side effect to the slot forces + // its value to be saved. + inline void pushLocalI32(uint32_t slot); + inline void pushLocalI64(uint32_t slot); + inline void pushLocalRef(uint32_t slot); + inline void pushLocalF64(uint32_t slot); + inline void pushLocalF32(uint32_t slot); +#ifdef ENABLE_WASM_SIMD + inline void pushLocalV128(uint32_t slot); +#endif + + // Push an U32 as an I64, zero-extending it in the process + inline void pushU32AsI64(RegI32 rs); + + ////////////////////////////////////////////////////////////////////// + // + // Value stack: poppers and peekers of values. + + // Pop some value off the stack. + inline AnyReg popAny(); + inline AnyReg popAny(AnyReg specific); + + // Call only from other popI32() variants. v must be the stack top. May pop + // the CPU stack. + inline void popI32(const Stk& v, RegI32 dest); + + [[nodiscard]] inline RegI32 popI32(); + inline RegI32 popI32(RegI32 specific); + +#ifdef ENABLE_WASM_SIMD + // Call only from other popV128() variants. v must be the stack top. May pop + // the CPU stack. + inline void popV128(const Stk& v, RegV128 dest); + + [[nodiscard]] inline RegV128 popV128(); + inline RegV128 popV128(RegV128 specific); +#endif + + // Call only from other popI64() variants. v must be the stack top. May pop + // the CPU stack. + inline void popI64(const Stk& v, RegI64 dest); + + [[nodiscard]] inline RegI64 popI64(); + inline RegI64 popI64(RegI64 specific); + + // Call only from other popRef() variants. v must be the stack top. May pop + // the CPU stack. + inline void popRef(const Stk& v, RegRef dest); + + inline RegRef popRef(RegRef specific); + [[nodiscard]] inline RegRef popRef(); + + // Call only from other popPtr() variants. v must be the stack top. May pop + // the CPU stack. + inline void popPtr(const Stk& v, RegPtr dest); + + inline RegPtr popPtr(RegPtr specific); + [[nodiscard]] inline RegPtr popPtr(); + + // Call only from other popF64() variants. v must be the stack top. May pop + // the CPU stack. + inline void popF64(const Stk& v, RegF64 dest); + + [[nodiscard]] inline RegF64 popF64(); + inline RegF64 popF64(RegF64 specific); + + // Call only from other popF32() variants. v must be the stack top. May pop + // the CPU stack. + inline void popF32(const Stk& v, RegF32 dest); + + [[nodiscard]] inline RegF32 popF32(); + inline RegF32 popF32(RegF32 specific); + + // Templated variation of the foregoing, for use by templated emitters. + template + inline RegType pop(); + + // Constant poppers will return true and pop the value if the stack top is a + // constant of the appropriate type; otherwise pop nothing and return false. + [[nodiscard]] inline bool hasConst() const; + [[nodiscard]] inline bool popConst(int32_t* c); + [[nodiscard]] inline bool popConst(int64_t* c); + [[nodiscard]] inline bool peekConst(int32_t* c); + [[nodiscard]] inline bool peekConst(int64_t* c); + [[nodiscard]] inline bool peek2xConst(int32_t* c0, int32_t* c1); + [[nodiscard]] inline bool popConstPositivePowerOfTwo(int32_t* c, + uint_fast8_t* power, + int32_t cutoff); + [[nodiscard]] inline bool popConstPositivePowerOfTwo(int64_t* c, + uint_fast8_t* power, + int64_t cutoff); + + // Shorthand: Pop r1, then r0. + inline void pop2xI32(RegI32* r0, RegI32* r1); + inline void pop2xI64(RegI64* r0, RegI64* r1); + inline void pop2xF32(RegF32* r0, RegF32* r1); + inline void pop2xF64(RegF64* r0, RegF64* r1); +#ifdef ENABLE_WASM_SIMD + inline void pop2xV128(RegV128* r0, RegV128* r1); +#endif + inline void pop2xRef(RegRef* r0, RegRef* r1); + + // Pop to a specific register + inline RegI32 popI32ToSpecific(RegI32 specific); + inline RegI64 popI64ToSpecific(RegI64 specific); + +#ifdef JS_CODEGEN_ARM + // Pop an I64 as a valid register pair. + inline RegI64 popI64Pair(); +#endif + + // Pop an I64 but narrow it and return the narrowed part. + inline RegI32 popI64ToI32(); + inline RegI32 popI64ToSpecificI32(RegI32 specific); + + // Pop the stack until it has the desired size, but do not move the physical + // stack pointer. + inline void popValueStackTo(uint32_t stackSize); + + // Pop the given number of elements off the value stack, but do not move + // the physical stack pointer. + inline void popValueStackBy(uint32_t items); + + // Peek into the stack at relativeDepth from the top. + inline Stk& peek(uint32_t relativeDepth); + + // Peek the reference value at the specified depth and load it into a + // register. + inline void peekRefAt(uint32_t depth, RegRef dest); + + // Peek at the value on the top of the stack and return true if it is a Local + // of any type. + [[nodiscard]] inline bool peekLocal(uint32_t* local); + + //////////////////////////////////////////////////////////////////////////// + // + // Block parameters and results. + // + // Blocks may have multiple parameters and multiple results. Blocks can also + // be the target of branches: the entry for loops, and the exit for + // non-loops. + // + // Passing multiple values to a non-branch target (i.e., the entry of a + // "block") falls out naturally: any items on the value stack can flow + // directly from one block to another. + // + // However, for branch targets, we need to allocate well-known locations for + // the branch values. The approach taken in the baseline compiler is to + // allocate registers to the top N values (currently N=1), and then stack + // locations for the rest. + // + + // Types of result registers that interest us for result-manipulating + // functions. + enum class ResultRegKind { + // General and floating result registers. + All, + + // General result registers only. + OnlyGPRs + }; + + // This is a flag ultimately intended for popBlockResults() that specifies how + // the CPU stack should be handled after the result values have been + // processed. + enum class ContinuationKind { + // Adjust the stack for a fallthrough: do nothing. + Fallthrough, + + // Adjust the stack for a jump: make the stack conform to the + // expected stack at the target + Jump + }; + + // TODO: It's definitely disputable whether the result register management is + // hot enough to warrant inlining at the outermost level. + + inline void needResultRegisters(ResultType type, ResultRegKind which); +#ifdef JS_64BIT + inline void widenInt32ResultRegisters(ResultType type); +#endif + inline void freeResultRegisters(ResultType type, ResultRegKind which); + inline void needIntegerResultRegisters(ResultType type); + inline void freeIntegerResultRegisters(ResultType type); + inline void needResultRegisters(ResultType type); + inline void freeResultRegisters(ResultType type); + void assertResultRegistersAvailable(ResultType type); + inline void captureResultRegisters(ResultType type); + inline void captureCallResultRegisters(ResultType type); + + void popRegisterResults(ABIResultIter& iter); + void popStackResults(ABIResultIter& iter, StackHeight stackBase); + + void popBlockResults(ResultType type, StackHeight stackBase, + ContinuationKind kind); + + // This function is similar to popBlockResults, but additionally handles the + // implicit exception pointer that is pushed to the value stack on entry to + // a catch handler by dropping it appropriately. + void popCatchResults(ResultType type, StackHeight stackBase); + + Stk captureStackResult(const ABIResult& result, StackHeight resultsBase, + uint32_t stackResultBytes); + + [[nodiscard]] bool pushResults(ResultType type, StackHeight resultsBase); + [[nodiscard]] bool pushBlockResults(ResultType type); + + // A combination of popBlockResults + pushBlockResults, used when entering a + // block with a control-flow join (loops) or split (if) to shuffle the + // fallthrough block parameters into the locations expected by the + // continuation. + // + // This function should only be called when entering a block with a + // control-flow join at the entry, where there are no live temporaries in + // the current block. + [[nodiscard]] bool topBlockParams(ResultType type); + + // A combination of popBlockResults + pushBlockResults, used before branches + // where we don't know the target (br_if / br_table). If and when the branch + // is taken, the stack results will be shuffled down into place. For br_if + // that has fallthrough, the parameters for the untaken branch flow through to + // the continuation. + [[nodiscard]] bool topBranchParams(ResultType type, StackHeight* height); + + // Conditional branches with fallthrough are preceded by a topBranchParams, so + // we know that there are no stack results that need to be materialized. In + // that case, we can just shuffle the whole block down before popping the + // stack. + void shuffleStackResultsBeforeBranch(StackHeight srcHeight, + StackHeight destHeight, ResultType type); + + ////////////////////////////////////////////////////////////////////// + // + // Stack maps + + // Various methods for creating a stackmap. Stackmaps are indexed by the + // lowest address of the instruction immediately *after* the instruction of + // interest. In practice that means either: the return point of a call, the + // instruction immediately after a trap instruction (the "resume" + // instruction), or the instruction immediately following a no-op (when + // debugging is enabled). + + // Create a vanilla stackmap. + [[nodiscard]] bool createStackMap(const char* who); + + // Create a stackmap as vanilla, but for a custom assembler offset. + [[nodiscard]] bool createStackMap(const char* who, + CodeOffset assemblerOffset); + + // Create a stack map as vanilla, and note the presence of a ref-typed + // DebugFrame on the stack. + [[nodiscard]] bool createStackMap( + const char* who, HasDebugFrameWithLiveRefs debugFrameWithLiveRefs); + + // The most general stackmap construction. + [[nodiscard]] bool createStackMap( + const char* who, const ExitStubMapVector& extras, + uint32_t assemblerOffset, + HasDebugFrameWithLiveRefs debugFrameWithLiveRefs); + + //////////////////////////////////////////////////////////// + // + // Control stack + + inline void initControl(Control& item, ResultType params); + inline Control& controlItem(); + inline Control& controlItem(uint32_t relativeDepth); + inline Control& controlOutermost(); + inline LabelKind controlKind(uint32_t relativeDepth); + + //////////////////////////////////////////////////////////// + // + // Debugger API + + // Insert a breakpoint almost anywhere. This will create a call, with all the + // overhead that entails. + void insertBreakablePoint(CallSiteDesc::Kind kind); + + // Insert code at the end of a function for breakpoint filtering. + void insertBreakpointStub(); + + // Debugger API used at the return point: shuffle register return values off + // to memory for the debugger to see; and get them back again. + void saveRegisterReturnValues(const ResultType& resultType); + void restoreRegisterReturnValues(const ResultType& resultType); + + ////////////////////////////////////////////////////////////////////// + // + // Function prologue and epilogue. + + // Set up and tear down frame, execute prologue and epilogue. + [[nodiscard]] bool beginFunction(); + [[nodiscard]] bool endFunction(); + + // Move return values to memory before returning, as appropriate + void popStackReturnValues(const ResultType& resultType); + + ////////////////////////////////////////////////////////////////////// + // + // Calls. + + void beginCall(FunctionCall& call, UseABI useABI, + RestoreRegisterStateAndRealm restoreRegisterStateAndRealm); + void endCall(FunctionCall& call, size_t stackSpace); + void startCallArgs(size_t stackArgAreaSizeUnaligned, FunctionCall* call); + ABIArg reservePointerArgument(FunctionCall* call); + void passArg(ValType type, const Stk& arg, FunctionCall* call); + CodeOffset callDefinition(uint32_t funcIndex, const FunctionCall& call); + CodeOffset callSymbolic(SymbolicAddress callee, const FunctionCall& call); + + // Precondition for the call*() methods: sync() + + bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex, + const Stk& indexVal, const FunctionCall& call, + CodeOffset* fastCallOffset, CodeOffset* slowCallOffset); + CodeOffset callImport(unsigned instanceDataOffset, const FunctionCall& call); +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + void callRef(const Stk& calleeRef, const FunctionCall& call, + CodeOffset* fastCallOffset, CodeOffset* slowCallOffset); +#endif + CodeOffset builtinCall(SymbolicAddress builtin, const FunctionCall& call); + CodeOffset builtinInstanceMethodCall(const SymbolicAddressSignature& builtin, + const ABIArg& instanceArg, + const FunctionCall& call); + [[nodiscard]] bool pushCallResults(const FunctionCall& call, ResultType type, + const StackResultsLoc& loc); + + // Helpers to pick up the returned value from the return register. + inline RegI32 captureReturnedI32(); + inline RegI64 captureReturnedI64(); + inline RegF32 captureReturnedF32(const FunctionCall& call); + inline RegF64 captureReturnedF64(const FunctionCall& call); +#ifdef ENABLE_WASM_SIMD + inline RegV128 captureReturnedV128(const FunctionCall& call); +#endif + inline RegRef captureReturnedRef(); + + ////////////////////////////////////////////////////////////////////// + // + // Register-to-register moves. These emit nothing if src == dest. + + inline void moveI32(RegI32 src, RegI32 dest); + inline void moveI64(RegI64 src, RegI64 dest); + inline void moveRef(RegRef src, RegRef dest); + inline void movePtr(RegPtr src, RegPtr dest); + inline void moveF64(RegF64 src, RegF64 dest); + inline void moveF32(RegF32 src, RegF32 dest); +#ifdef ENABLE_WASM_SIMD + inline void moveV128(RegV128 src, RegV128 dest); +#endif + + template + inline void move(RegType src, RegType dest); + + ////////////////////////////////////////////////////////////////////// + // + // Immediate-to-register moves. + // + // The compiler depends on moveImm32() clearing the high bits of a 64-bit + // register on 64-bit systems except MIPS64 And LoongArch64 where high bits + // are sign extended from lower bits, see doc block "64-bit GPRs carrying + // 32-bit values" in MacroAssembler.h. + + inline void moveImm32(int32_t v, RegI32 dest); + inline void moveImm64(int64_t v, RegI64 dest); + inline void moveImmRef(intptr_t v, RegRef dest); + + ////////////////////////////////////////////////////////////////////// + // + // Sundry low-level code generators. + + // Check the interrupt flag, trap if it is set. + [[nodiscard]] bool addInterruptCheck(); + + // Check that the value is not zero, trap if it is. + void checkDivideByZero(RegI32 rhs); + void checkDivideByZero(RegI64 r); + + // Check that a signed division will not overflow, trap or flush-to-zero if it + // will according to `zeroOnOverflow`. + void checkDivideSignedOverflow(RegI32 rhs, RegI32 srcDest, Label* done, + bool zeroOnOverflow); + void checkDivideSignedOverflow(RegI64 rhs, RegI64 srcDest, Label* done, + bool zeroOnOverflow); + + // Emit a jump table to be used by tableSwitch() + void jumpTable(const LabelVector& labels, Label* theTable); + + // Emit a table switch, `theTable` is the jump table. + void tableSwitch(Label* theTable, RegI32 switchValue, Label* dispatchCode); + + // Compare i64 and set an i32 boolean result according to the condition. + inline void cmp64Set(Assembler::Condition cond, RegI64 lhs, RegI64 rhs, + RegI32 dest); + + // Round floating to integer. + [[nodiscard]] inline bool supportsRoundInstruction(RoundingMode mode); + inline void roundF32(RoundingMode roundingMode, RegF32 f0); + inline void roundF64(RoundingMode roundingMode, RegF64 f0); + + // These are just wrappers around assembler functions, but without + // type-specific names, and using our register abstractions for better type + // discipline. + inline void branchTo(Assembler::DoubleCondition c, RegF64 lhs, RegF64 rhs, + Label* l); + inline void branchTo(Assembler::DoubleCondition c, RegF32 lhs, RegF32 rhs, + Label* l); + inline void branchTo(Assembler::Condition c, RegI32 lhs, RegI32 rhs, + Label* l); + inline void branchTo(Assembler::Condition c, RegI32 lhs, Imm32 rhs, Label* l); + inline void branchTo(Assembler::Condition c, RegI64 lhs, RegI64 rhs, + Label* l); + inline void branchTo(Assembler::Condition c, RegI64 lhs, Imm64 rhs, Label* l); + inline void branchTo(Assembler::Condition c, RegRef lhs, ImmWord rhs, + Label* l); + +#ifdef JS_CODEGEN_X86 + // Store r in instance scratch storage after first loading the instance from + // the frame into the regForInstance. regForInstance must be neither of the + // registers in r. + void stashI64(RegPtr regForInstance, RegI64 r); + + // Load r from the instance scratch storage after first loading the instance + // from the frame into the regForInstance. regForInstance can be one of the + // registers in r. + void unstashI64(RegPtr regForInstance, RegI64 r); +#endif + + ////////////////////////////////////////////////////////////////////// + // + // Code generators for actual operations. + + template + void quotientOrRemainder(RegType rs, RegType rsd, RegType reserved, + IsUnsigned isUnsigned, ZeroOnOverflow zeroOnOverflow, + bool isConst, IntType c, + void (*operate)(MacroAssembler&, RegType, RegType, + RegType, IsUnsigned)); + + [[nodiscard]] bool truncateF32ToI32(RegF32 src, RegI32 dest, + TruncFlags flags); + [[nodiscard]] bool truncateF64ToI32(RegF64 src, RegI32 dest, + TruncFlags flags); + +#ifndef RABALDR_FLOAT_TO_I64_CALLOUT + [[nodiscard]] RegF64 needTempForFloatingToI64(TruncFlags flags); + [[nodiscard]] bool truncateF32ToI64(RegF32 src, RegI64 dest, TruncFlags flags, + RegF64 temp); + [[nodiscard]] bool truncateF64ToI64(RegF64 src, RegI64 dest, TruncFlags flags, + RegF64 temp); +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT + [[nodiscard]] RegI32 needConvertI64ToFloatTemp(ValType to, bool isUnsigned); + void convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest, RegI32 temp); + void convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest, RegI32 temp); +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + + ////////////////////////////////////////////////////////////////////// + // + // Global variable access. + + Address addressOfGlobalVar(const GlobalDesc& global, RegPtr tmp); + + ////////////////////////////////////////////////////////////////////// + // + // Table access. + + Address addressOfTableField(uint32_t tableIndex, uint32_t fieldOffset, + RegPtr instance); + void loadTableLength(uint32_t tableIndex, RegPtr instance, RegI32 length); + void loadTableElements(uint32_t tableIndex, RegPtr instance, RegPtr elements); + + ////////////////////////////////////////////////////////////////////// + // + // Heap access. + + void bceCheckLocal(MemoryAccessDesc* access, AccessCheck* check, + uint32_t local); + void bceLocalIsUpdated(uint32_t local); + + // Fold offsets into ptr and bounds check as necessary. The instance will be + // valid in cases where it's needed. + template + void prepareMemoryAccess(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegIndexType ptr); + + void branchAddNoOverflow(uint64_t offset, RegI32 ptr, Label* ok); + void branchTestLowZero(RegI32 ptr, Imm32 mask, Label* ok); + void boundsCheck4GBOrLargerAccess(RegPtr instance, RegI32 ptr, Label* ok); + void boundsCheckBelow4GBAccess(RegPtr instance, RegI32 ptr, Label* ok); + + void branchAddNoOverflow(uint64_t offset, RegI64 ptr, Label* ok); + void branchTestLowZero(RegI64 ptr, Imm32 mask, Label* ok); + void boundsCheck4GBOrLargerAccess(RegPtr instance, RegI64 ptr, Label* ok); + void boundsCheckBelow4GBAccess(RegPtr instance, RegI64 ptr, Label* ok); + +#if defined(WASM_HAS_HEAPREG) + template + BaseIndex prepareAtomicMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, RegPtr instance, + RegIndexType ptr); +#else + // Some consumers depend on the returned Address not incorporating instance, + // as instance may be the scratch register. + template + Address prepareAtomicMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, RegPtr instance, + RegIndexType ptr); +#endif + + template + void computeEffectiveAddress(MemoryAccessDesc* access); + + [[nodiscard]] bool needInstanceForAccess(const AccessCheck& check); + + // ptr and dest may be the same iff dest is I32. + // This may destroy ptr even if ptr and dest are not the same. + void executeLoad(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI32 ptr, AnyReg dest, RegI32 temp); + void load(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, + RegI32 ptr, AnyReg dest, RegI32 temp); +#ifdef ENABLE_WASM_MEMORY64 + void load(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, + RegI64 ptr, AnyReg dest, RegI64 temp); +#endif + + template + void doLoadCommon(MemoryAccessDesc* access, AccessCheck check, ValType type); + + void loadCommon(MemoryAccessDesc* access, AccessCheck check, ValType type); + + // ptr and src must not be the same register. + // This may destroy ptr and src. + void executeStore(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI32 ptr, AnyReg src, RegI32 temp); + void store(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, + RegI32 ptr, AnyReg src, RegI32 temp); +#ifdef ENABLE_WASM_MEMORY64 + void store(MemoryAccessDesc* access, AccessCheck* check, RegPtr instance, + RegI64 ptr, AnyReg src, RegI64 temp); +#endif + + template + void doStoreCommon(MemoryAccessDesc* access, AccessCheck check, + ValType resultType); + + void storeCommon(MemoryAccessDesc* access, AccessCheck check, + ValType resultType); + + void atomicLoad(MemoryAccessDesc* access, ValType type); +#if !defined(JS_64BIT) + template + void atomicLoad64(MemoryAccessDesc* desc); +#endif + + void atomicStore(MemoryAccessDesc* access, ValType type); + + void atomicRMW(MemoryAccessDesc* access, ValType type, AtomicOp op); + template + void atomicRMW32(MemoryAccessDesc* access, ValType type, AtomicOp op); + template + void atomicRMW64(MemoryAccessDesc* access, ValType type, AtomicOp op); + + void atomicXchg(MemoryAccessDesc* access, ValType type); + template + void atomicXchg64(MemoryAccessDesc* access, WantResult wantResult); + template + void atomicXchg32(MemoryAccessDesc* access, ValType type); + + void atomicCmpXchg(MemoryAccessDesc* access, ValType type); + template + void atomicCmpXchg32(MemoryAccessDesc* access, ValType type); + template + void atomicCmpXchg64(MemoryAccessDesc* access, ValType type); + + template + RegType popConstMemoryAccess(MemoryAccessDesc* access, AccessCheck* check); + template + RegType popMemoryAccess(MemoryAccessDesc* access, AccessCheck* check); + + void pushHeapBase(); + + //////////////////////////////////////////////////////////////////////////// + // + // Platform-specific popping and register targeting. + + // The simple popping methods pop values into targeted registers; the caller + // can free registers using standard functions. These are always called + // popXForY where X says something about types and Y something about the + // operation being targeted. + + RegI32 needRotate64Temp(); + void popAndAllocateForDivAndRemI32(RegI32* r0, RegI32* r1, RegI32* reserved); + void popAndAllocateForMulI64(RegI64* r0, RegI64* r1, RegI32* temp); +#ifndef RABALDR_INT_DIV_I64_CALLOUT + void popAndAllocateForDivAndRemI64(RegI64* r0, RegI64* r1, RegI64* reserved, + IsRemainder isRemainder); +#endif + RegI32 popI32RhsForShift(); + RegI32 popI32RhsForShiftI64(); + RegI64 popI64RhsForShift(); + RegI32 popI32RhsForRotate(); + RegI64 popI64RhsForRotate(); + void popI32ForSignExtendI64(RegI64* r0); + void popI64ForSignExtendI64(RegI64* r0); + + //////////////////////////////////////////////////////////// + // + // Sundry helpers. + + // Retrieve the current bytecodeOffset. + inline BytecodeOffset bytecodeOffset() const; + + // Generate a trap instruction for the current bytecodeOffset. + inline void trap(Trap t) const; + + // Abstracted helper for throwing, used for throw, rethrow, and rethrowing + // at the end of a series of catch blocks (if none matched the exception). + [[nodiscard]] bool throwFrom(RegRef exn); + + // Load the specified tag object from the Instance. + void loadTag(RegPtr instanceData, uint32_t tagIndex, RegRef tagDst); + + // Load the pending exception state from the Instance and then reset it. + void consumePendingException(RegRef* exnDst, RegRef* tagDst); + + [[nodiscard]] bool startTryNote(size_t* tryNoteIndex); + void finishTryNote(size_t tryNoteIndex); + + //////////////////////////////////////////////////////////// + // + // Barriers support. + + // This emits a GC pre-write barrier. The pre-barrier is needed when we + // replace a member field with a new value, and the previous field value + // might have no other referents, and incremental GC is ongoing. The field + // might belong to an object or be a stack slot or a register or a heap + // allocated value. + // + // let obj = { field: previousValue }; + // obj.field = newValue; // previousValue must be marked with a pre-barrier. + // + // The `valueAddr` is the address of the location that we are about to + // update. This function preserves that register. + void emitPreBarrier(RegPtr valueAddr); + + // This emits a GC post-write barrier. The post-barrier is needed when we + // replace a member field with a new value, the new value is in the nursery, + // and the containing object is a tenured object. The field must then be + // added to the store buffer so that the nursery can be correctly collected. + // The field might belong to an object or be a stack slot or a register or a + // heap allocated value. + // + // For the difference between 'precise' and 'imprecise', look at the + // documentation on PostBarrierKind. + // + // `object` is a pointer to the object that contains the field. It is used, if + // present, to skip adding a store buffer entry when the containing object is + // in the nursery. This register is preserved by this function. + // `valueAddr` is the address of the location that we are writing to. This + // register is consumed by this function. + // `prevValue` is the value that existed in the field before `value` was + // stored. This register is consumed by this function. + // `value` is the value that was stored in the field. This register is + // preserved by this function. + [[nodiscard]] bool emitPostBarrierImprecise(const Maybe& object, + RegPtr valueAddr, RegRef value); + [[nodiscard]] bool emitPostBarrierPrecise(const Maybe& object, + RegPtr valueAddr, RegRef prevValue, + RegRef value); + + // Emits a store to a JS object pointer at the address `valueAddr`, which is + // inside the GC cell `object`. + // + // Preserves `object` and `value`. Consumes `valueAddr`. + [[nodiscard]] bool emitBarrieredStore(const Maybe& object, + RegPtr valueAddr, RegRef value, + PreBarrierKind preBarrierKind, + PostBarrierKind postBarrierKind); + + // Emits a store of nullptr to a JS object pointer at the address valueAddr. + // Preserves `valueAddr`. + void emitBarrieredClear(RegPtr valueAddr); + + //////////////////////////////////////////////////////////// + // + // Machinery for optimized conditional branches. See comments in the + // implementation. + + void setLatentCompare(Assembler::Condition compareOp, ValType operandType); + void setLatentCompare(Assembler::DoubleCondition compareOp, + ValType operandType); + void setLatentEqz(ValType operandType); + bool hasLatentOp() const; + void resetLatentOp(); + // Jump to the given branch, passing results, if the condition, `cond` + // matches between `lhs` and `rhs. + template + [[nodiscard]] bool jumpConditionalWithResults(BranchState* b, Cond cond, + Lhs lhs, Rhs rhs); +#ifdef ENABLE_WASM_GC + // Jump to the given branch, passing results, if the WasmGcObject, `object`, + // is a subtype of `destType`. + [[nodiscard]] bool jumpConditionalWithResults(BranchState* b, RegRef object, + RefType sourceType, + RefType destType, + bool onSuccess); +#endif + template + [[nodiscard]] bool sniffConditionalControlCmp(Cond compareOp, + ValType operandType); + [[nodiscard]] bool sniffConditionalControlEqz(ValType operandType); + void emitBranchSetup(BranchState* b); + [[nodiscard]] bool emitBranchPerform(BranchState* b); + + ////////////////////////////////////////////////////////////////////// + + [[nodiscard]] bool emitBody(); + [[nodiscard]] bool emitBlock(); + [[nodiscard]] bool emitLoop(); + [[nodiscard]] bool emitIf(); + [[nodiscard]] bool emitElse(); + // Used for common setup for catch and catch_all. + void emitCatchSetup(LabelKind kind, Control& tryCatch, + const ResultType& resultType); + // Helper function used to generate landing pad code for the special + // case in which `delegate` jumps to a function's body block. + [[nodiscard]] bool emitBodyDelegateThrowPad(); + + [[nodiscard]] bool emitTry(); + [[nodiscard]] bool emitCatch(); + [[nodiscard]] bool emitCatchAll(); + [[nodiscard]] bool emitDelegate(); + [[nodiscard]] bool emitThrow(); + [[nodiscard]] bool emitRethrow(); + [[nodiscard]] bool emitEnd(); + [[nodiscard]] bool emitBr(); + [[nodiscard]] bool emitBrIf(); + [[nodiscard]] bool emitBrTable(); + [[nodiscard]] bool emitDrop(); + [[nodiscard]] bool emitReturn(); + + // A flag passed to emitCallArgs, describing how the value stack is laid out. + enum class CalleeOnStack { + // After the arguments to the call, there is a callee pushed onto value + // stack. This is only the case for callIndirect. To get the arguments to + // the call, emitCallArgs has to reach one element deeper into the value + // stack, to skip the callee. + True, + + // No callee on the stack. + False + }; + + [[nodiscard]] bool emitCallArgs(const ValTypeVector& argTypes, + const StackResultsLoc& results, + FunctionCall* baselineCall, + CalleeOnStack calleeOnStack); + + [[nodiscard]] bool emitCall(); + [[nodiscard]] bool emitCallIndirect(); + [[nodiscard]] bool emitUnaryMathBuiltinCall(SymbolicAddress callee, + ValType operandType); + [[nodiscard]] bool emitGetLocal(); + [[nodiscard]] bool emitSetLocal(); + [[nodiscard]] bool emitTeeLocal(); + [[nodiscard]] bool emitGetGlobal(); + [[nodiscard]] bool emitSetGlobal(); + [[nodiscard]] RegPtr maybeLoadInstanceForAccess(const AccessCheck& check); + [[nodiscard]] RegPtr maybeLoadInstanceForAccess(const AccessCheck& check, + RegPtr specific); + [[nodiscard]] bool emitLoad(ValType type, Scalar::Type viewType); + [[nodiscard]] bool emitStore(ValType resultType, Scalar::Type viewType); + [[nodiscard]] bool emitSelect(bool typed); + + template + [[nodiscard]] bool emitSetOrTeeLocal(uint32_t slot); + + [[nodiscard]] bool endBlock(ResultType type); + [[nodiscard]] bool endIfThen(ResultType type); + [[nodiscard]] bool endIfThenElse(ResultType type); + [[nodiscard]] bool endTryCatch(ResultType type); + + void doReturn(ContinuationKind kind); + void pushReturnValueOfCall(const FunctionCall& call, MIRType type); + + [[nodiscard]] bool pushStackResultsForCall(const ResultType& type, + RegPtr temp, StackResultsLoc* loc); + void popStackResultsAfterCall(const StackResultsLoc& results, + uint32_t stackArgBytes); + + void emitCompareI32(Assembler::Condition compareOp, ValType compareType); + void emitCompareI64(Assembler::Condition compareOp, ValType compareType); + void emitCompareF32(Assembler::DoubleCondition compareOp, + ValType compareType); + void emitCompareF64(Assembler::DoubleCondition compareOp, + ValType compareType); + void emitCompareRef(Assembler::Condition compareOp, ValType compareType); + + template + inline CompilerType& selectCompiler(); + + template + inline void emitUnop(void (*op)(MacroAssembler& masm, SourceType rs, + DestType rd)); + + template + inline void emitUnop(void (*op)(MacroAssembler& masm, SourceType rs, + DestType rd, TempType temp)); + + template + inline void emitUnop(ImmType immediate, void (*op)(MacroAssembler&, ImmType, + SourceType, DestType)); + + template + inline void emitUnop(void (*op)(CompilerType& compiler, RegType rsd)); + + template + inline void emitUnop(void (*op)(BaseCompiler& bc, RegType rsd, TempType rt), + TempType (*getSpecializedTemp)(BaseCompiler& bc)); + + template + inline void emitBinop(void (*op)(CompilerType& masm, RhsType src, + LhsDestType srcDest)); + + template + inline void emitBinop(void (*op)(MacroAssembler& masm, RhsDestType src, + LhsType srcDest, RhsDestOp)); + + template + inline void emitBinop(void (*)(MacroAssembler& masm, RhsType rs, + LhsDestType rsd, TempType temp)); + + template + inline void emitBinop(void (*)(MacroAssembler& masm, RhsType rs, + LhsDestType rsd, TempType1 temp1, + TempType2 temp2)); + + template + inline void emitBinop(ImmType immediate, void (*op)(MacroAssembler&, ImmType, + RhsType, LhsDestType)); + + template + inline void emitBinop(ImmType immediate, + void (*op)(MacroAssembler&, ImmType, RhsType, + LhsDestType, TempType1 temp1, + TempType2 temp2)); + + template + inline void emitBinop(void (*op)(CompilerType1& compiler1, RegType rs, + RegType rd), + void (*opConst)(CompilerType2& compiler2, ImmType c, + RegType rd), + RegType (BaseCompiler::*rhsPopper)() = nullptr); + + template + inline void emitTernary(void (*op)(CompilerType&, ValType src0, ValType src1, + ValType srcDest)); + + template + inline void emitTernary(void (*op)(CompilerType&, ValType src0, ValType src1, + ValType srcDest, ValType temp)); + + template + inline void emitTernaryResultLast(void (*op)(CompilerType&, ValType src0, + ValType src1, ValType srcDest)); + + template + [[nodiscard]] inline bool emitInstanceCallOp( + const SymbolicAddressSignature& fn, R reader); + + template + [[nodiscard]] inline bool emitInstanceCallOp( + const SymbolicAddressSignature& fn, R reader); + + template + [[nodiscard]] inline bool emitInstanceCallOp( + const SymbolicAddressSignature& fn, R reader); + + void emitMultiplyI64(); + void emitQuotientI32(); + void emitQuotientU32(); + void emitRemainderI32(); + void emitRemainderU32(); +#ifdef RABALDR_INT_DIV_I64_CALLOUT + [[nodiscard]] bool emitDivOrModI64BuiltinCall(SymbolicAddress callee, + ValType operandType); +#else + void emitQuotientI64(); + void emitQuotientU64(); + void emitRemainderI64(); + void emitRemainderU64(); +#endif + void emitRotrI64(); + void emitRotlI64(); + void emitEqzI32(); + void emitEqzI64(); + template + [[nodiscard]] bool emitTruncateF32ToI32(); + template + [[nodiscard]] bool emitTruncateF64ToI32(); +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + [[nodiscard]] bool emitConvertFloatingToInt64Callout(SymbolicAddress callee, + ValType operandType, + ValType resultType); +#else + template + [[nodiscard]] bool emitTruncateF32ToI64(); + template + [[nodiscard]] bool emitTruncateF64ToI64(); +#endif + void emitExtendI64_8(); + void emitExtendI64_16(); + void emitExtendI64_32(); + void emitExtendI32ToI64(); + void emitExtendU32ToI64(); +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + [[nodiscard]] bool emitConvertInt64ToFloatingCallout(SymbolicAddress callee, + ValType operandType, + ValType resultType); +#else + void emitConvertU64ToF32(); + void emitConvertU64ToF64(); +#endif + void emitRound(RoundingMode roundingMode, ValType operandType); + + // Generate a call to the instance function denoted by `builtin`, passing as + // args the top elements of the compiler's value stack and optionally an + // Instance* too. The relationship between the top of stack and arg + // ordering is as follows. If the value stack looks like this: + // + // A <- least recently pushed + // B + // C <- most recently pushed + // + // then the called function is expected to have signature [if an Instance* + // is also to be passed]: + // + // static Instance::foo(Instance*, A, B, C) + // + // and the SymbolicAddressSignature::argTypes array will be + // + // {_PTR, _A, _B, _C, _END} // _PTR is for the Instance* + // + // (see WasmBuiltins.cpp). In short, the most recently pushed value is the + // rightmost argument to the function. + [[nodiscard]] bool emitInstanceCall(const SymbolicAddressSignature& builtin); + + [[nodiscard]] bool emitMemoryGrow(); + [[nodiscard]] bool emitMemorySize(); + + [[nodiscard]] bool emitRefFunc(); + [[nodiscard]] bool emitRefNull(); + [[nodiscard]] bool emitRefIsNull(); +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + [[nodiscard]] bool emitRefAsNonNull(); + [[nodiscard]] bool emitBrOnNull(); + [[nodiscard]] bool emitBrOnNonNull(); + [[nodiscard]] bool emitCallRef(); +#endif + + [[nodiscard]] bool emitAtomicCmpXchg(ValType type, Scalar::Type viewType); + [[nodiscard]] bool emitAtomicLoad(ValType type, Scalar::Type viewType); + [[nodiscard]] bool emitAtomicRMW(ValType type, Scalar::Type viewType, + AtomicOp op); + [[nodiscard]] bool emitAtomicStore(ValType type, Scalar::Type viewType); + [[nodiscard]] bool emitWait(ValType type, uint32_t byteSize); + [[nodiscard]] bool atomicWait(ValType type, MemoryAccessDesc* access); + [[nodiscard]] bool emitWake(); + [[nodiscard]] bool atomicWake(MemoryAccessDesc* access); + [[nodiscard]] bool emitFence(); + [[nodiscard]] bool emitAtomicXchg(ValType type, Scalar::Type viewType); + [[nodiscard]] bool emitMemInit(); + [[nodiscard]] bool emitMemCopy(); + [[nodiscard]] bool memCopyCall(); + void memCopyInlineM32(); + [[nodiscard]] bool emitTableCopy(); + [[nodiscard]] bool emitDataOrElemDrop(bool isData); + [[nodiscard]] bool emitMemFill(); + [[nodiscard]] bool memFillCall(); + void memFillInlineM32(); + [[nodiscard]] bool emitTableInit(); + [[nodiscard]] bool emitTableFill(); + [[nodiscard]] bool emitMemDiscard(); + [[nodiscard]] bool emitTableGet(); + [[nodiscard]] bool emitTableGrow(); + [[nodiscard]] bool emitTableSet(); + [[nodiscard]] bool emitTableSize(); + + void emitTableBoundsCheck(uint32_t tableIndex, RegI32 index, RegPtr instance); + [[nodiscard]] bool emitTableGetAnyRef(uint32_t tableIndex); + [[nodiscard]] bool emitTableSetAnyRef(uint32_t tableIndex); + +#ifdef ENABLE_WASM_GC + [[nodiscard]] bool emitStructNew(); + [[nodiscard]] bool emitStructNewDefault(); + [[nodiscard]] bool emitStructGet(FieldWideningOp wideningOp); + [[nodiscard]] bool emitStructSet(); + [[nodiscard]] bool emitArrayNew(); + [[nodiscard]] bool emitArrayNewFixed(); + [[nodiscard]] bool emitArrayNewDefault(); + [[nodiscard]] bool emitArrayNewData(); + [[nodiscard]] bool emitArrayNewElem(); + [[nodiscard]] bool emitArrayGet(FieldWideningOp wideningOp); + [[nodiscard]] bool emitArraySet(); + [[nodiscard]] bool emitArrayLen(bool decodeIgnoredTypeIndex); + [[nodiscard]] bool emitArrayCopy(); + [[nodiscard]] bool emitRefTestV5(); + [[nodiscard]] bool emitRefCastV5(); + [[nodiscard]] bool emitBrOnCastV5(bool onSuccess); + [[nodiscard]] bool emitBrOnCastHeapV5(bool onSuccess, bool nullable); + [[nodiscard]] bool emitRefAsStructV5(); + [[nodiscard]] bool emitBrOnNonStructV5(); + [[nodiscard]] bool emitRefTest(bool nullable); + [[nodiscard]] bool emitRefCast(bool nullable); + [[nodiscard]] bool emitBrOnCastCommon(bool onSuccess, + uint32_t labelRelativeDepth, + const ResultType& labelType, + RefType sourceType, RefType destType); + [[nodiscard]] bool emitBrOnCast(); + [[nodiscard]] bool emitExternInternalize(); + [[nodiscard]] bool emitExternExternalize(); + + // Utility classes/methods to add trap information related to + // null pointer derefences/accesses. + struct NoNullCheck { + static void emitNullCheck(BaseCompiler*, RegRef) {} + static void emitTrapSite(BaseCompiler*) {} + }; + struct SignalNullCheck { + static void emitNullCheck(BaseCompiler* bc, RegRef rp); + static void emitTrapSite(BaseCompiler* bc); + }; + + // Load a pointer to the TypeDefInstanceData for a given type index + RegPtr loadTypeDefInstanceData(uint32_t typeIndex); + // Load a pointer to the SuperTypeVector for a given type index + RegPtr loadSuperTypeVector(uint32_t typeIndex); + + RegPtr emitGcArrayGetData(RegRef rp); + template + RegI32 emitGcArrayGetNumElements(RegRef rp); + void emitGcArrayBoundsCheck(RegI32 index, RegI32 numElements); + template + void emitGcGet(FieldType type, FieldWideningOp wideningOp, const T& src); + template + void emitGcSetScalar(const T& dst, FieldType type, AnyReg value); + + // Common code for both old and new ref.test instructions. + void emitRefTestCommon(RefType sourceType, RefType destType); + // Common code for both old and new ref.cast instructions. + void emitRefCastCommon(RefType sourceType, RefType destType); + + // Allocate registers and branch if the given object is a subtype of the given + // heap type. + void branchGcRefType(RegRef object, RefType sourceType, RefType destType, + Label* label, bool onSuccess); + + // Write `value` to wasm struct `object`, at `areaBase + areaOffset`. The + // caller must decide on the in- vs out-of-lineness before the call and set + // the latter two accordingly; this routine does not take that into account. + // The value in `object` is unmodified, but `areaBase` and `value` may get + // trashed. + template + [[nodiscard]] bool emitGcStructSet(RegRef object, RegPtr areaBase, + uint32_t areaOffset, FieldType fieldType, + AnyReg value, + PreBarrierKind preBarrierKind); + + [[nodiscard]] bool emitGcArraySet(RegRef object, RegPtr data, RegI32 index, + const ArrayType& array, AnyReg value, + PreBarrierKind preBarrierKind); +#endif // ENABLE_WASM_GC + +#ifdef ENABLE_WASM_SIMD + void emitVectorAndNot(); +# ifdef ENABLE_WASM_RELAXED_SIMD + void emitDotI8x16I7x16AddS(); +# endif + + void loadSplat(MemoryAccessDesc* access); + void loadZero(MemoryAccessDesc* access); + void loadExtend(MemoryAccessDesc* access, Scalar::Type viewType); + void loadLane(MemoryAccessDesc* access, uint32_t laneIndex); + void storeLane(MemoryAccessDesc* access, uint32_t laneIndex); + + [[nodiscard]] bool emitLoadSplat(Scalar::Type viewType); + [[nodiscard]] bool emitLoadZero(Scalar::Type viewType); + [[nodiscard]] bool emitLoadExtend(Scalar::Type viewType); + [[nodiscard]] bool emitLoadLane(uint32_t laneSize); + [[nodiscard]] bool emitStoreLane(uint32_t laneSize); + [[nodiscard]] bool emitVectorShuffle(); + [[nodiscard]] bool emitVectorLaneSelect(); +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + [[nodiscard]] bool emitVectorShiftRightI64x2(); +# endif +#endif + [[nodiscard]] bool emitIntrinsic(); +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_object_h diff --git a/js/src/wasm/WasmBCCodegen-inl.h b/js/src/wasm/WasmBCCodegen-inl.h new file mode 100644 index 0000000000..03d5b9f078 --- /dev/null +++ b/js/src/wasm/WasmBCCodegen-inl.h @@ -0,0 +1,524 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: inline methods in the +// compiler for basic code generation. + +#ifndef wasm_wasm_baseline_codegen_inl_h +#define wasm_wasm_baseline_codegen_inl_h + +// The templates for register management must be defined by the time we use the +// templated emitters, below. +#include "wasm/WasmBCRegMgmt-inl.h" + +namespace js { +namespace wasm { + +////////////////////////////////////////////////////////////////////////////// +// +// Register-to-register moves. + +void BaseCompiler::moveI32(RegI32 src, RegI32 dest) { + if (src != dest) { + masm.move32(src, dest); + } +} + +void BaseCompiler::moveI64(RegI64 src, RegI64 dest) { + if (src != dest) { + masm.move64(src, dest); + } +} + +void BaseCompiler::moveRef(RegRef src, RegRef dest) { + if (src != dest) { + masm.movePtr(src, dest); + } +} + +void BaseCompiler::movePtr(RegPtr src, RegPtr dest) { + if (src != dest) { + masm.movePtr(src, dest); + } +} + +void BaseCompiler::moveF64(RegF64 src, RegF64 dest) { + if (src != dest) { + masm.moveDouble(src, dest); + } +} + +void BaseCompiler::moveF32(RegF32 src, RegF32 dest) { + if (src != dest) { + masm.moveFloat32(src, dest); + } +} + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::moveV128(RegV128 src, RegV128 dest) { + if (src != dest) { + masm.moveSimd128(src, dest); + } +} +#endif + +template <> +inline void BaseCompiler::move(RegI32 src, RegI32 dest) { + moveI32(src, dest); +} + +template <> +inline void BaseCompiler::move(RegI64 src, RegI64 dest) { + moveI64(src, dest); +} + +template <> +inline void BaseCompiler::move(RegF32 src, RegF32 dest) { + moveF32(src, dest); +} + +template <> +inline void BaseCompiler::move(RegF64 src, RegF64 dest) { + moveF64(src, dest); +} + +template <> +inline void BaseCompiler::move(RegRef src, RegRef dest) { + moveRef(src, dest); +} + +template <> +inline void BaseCompiler::move(RegPtr src, RegPtr dest) { + movePtr(src, dest); +} + +#ifdef ENABLE_WASM_SIMD +template <> +inline void BaseCompiler::move(RegV128 src, RegV128 dest) { + moveV128(src, dest); +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Constant loads. + +void BaseCompiler::moveImm32(int32_t v, RegI32 dest) { + masm.move32(Imm32(v), dest); +} + +void BaseCompiler::moveImm64(int64_t v, RegI64 dest) { + masm.move64(Imm64(v), dest); +} + +void BaseCompiler::moveImmRef(intptr_t v, RegRef dest) { + masm.movePtr(ImmWord(v), dest); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Calls. + +RegI32 BaseCompiler::captureReturnedI32() { + RegI32 r = RegI32(ReturnReg); + MOZ_ASSERT(isAvailableI32(r)); + needI32(r); +#if defined(JS_64BIT) + masm.widenInt32(r); +#endif + return r; +} + +RegI64 BaseCompiler::captureReturnedI64() { + RegI64 r = RegI64(ReturnReg64); + MOZ_ASSERT(isAvailableI64(r)); + needI64(r); + return r; +} + +RegF32 BaseCompiler::captureReturnedF32(const FunctionCall& call) { + RegF32 r = RegF32(ReturnFloat32Reg); + MOZ_ASSERT(isAvailableF32(r)); + needF32(r); +#if defined(JS_CODEGEN_ARM) + if (call.usesSystemAbi && !call.hardFP) { + masm.ma_vxfer(ReturnReg, r); + } +#endif + return r; +} + +RegF64 BaseCompiler::captureReturnedF64(const FunctionCall& call) { + RegF64 r = RegF64(ReturnDoubleReg); + MOZ_ASSERT(isAvailableF64(r)); + needF64(r); +#if defined(JS_CODEGEN_ARM) + if (call.usesSystemAbi && !call.hardFP) { + masm.ma_vxfer(ReturnReg64.low, ReturnReg64.high, r); + } +#endif + return r; +} + +#ifdef ENABLE_WASM_SIMD +RegV128 BaseCompiler::captureReturnedV128(const FunctionCall& call) { + RegV128 r = RegV128(ReturnSimd128Reg); + MOZ_ASSERT(isAvailableV128(r)); + needV128(r); + return r; +} +#endif + +RegRef BaseCompiler::captureReturnedRef() { + RegRef r = RegRef(ReturnReg); + MOZ_ASSERT(isAvailableRef(r)); + needRef(r); + return r; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Miscellaneous. + +void BaseCompiler::trap(Trap t) const { masm.wasmTrap(t, bytecodeOffset()); } + +void BaseCompiler::cmp64Set(Assembler::Condition cond, RegI64 lhs, RegI64 rhs, + RegI32 dest) { +#if defined(JS_PUNBOX64) + masm.cmpPtrSet(cond, lhs.reg, rhs.reg, dest); +#else + // TODO / OPTIMIZE (Bug 1316822): This is pretty branchy, we should be + // able to do better. + Label done, condTrue; + masm.branch64(cond, lhs, rhs, &condTrue); + moveImm32(0, dest); + masm.jump(&done); + masm.bind(&condTrue); + moveImm32(1, dest); + masm.bind(&done); +#endif +} + +[[nodiscard]] bool BaseCompiler::supportsRoundInstruction(RoundingMode mode) { + return Assembler::HasRoundInstruction(mode); +} + +void BaseCompiler::roundF32(RoundingMode roundingMode, RegF32 f0) { + masm.nearbyIntFloat32(roundingMode, f0, f0); +} + +void BaseCompiler::roundF64(RoundingMode roundingMode, RegF64 f0) { + masm.nearbyIntDouble(roundingMode, f0, f0); +} + +void BaseCompiler::branchTo(Assembler::DoubleCondition c, RegF64 lhs, + RegF64 rhs, Label* l) { + masm.branchDouble(c, lhs, rhs, l); +} + +void BaseCompiler::branchTo(Assembler::DoubleCondition c, RegF32 lhs, + RegF32 rhs, Label* l) { + masm.branchFloat(c, lhs, rhs, l); +} + +void BaseCompiler::branchTo(Assembler::Condition c, RegI32 lhs, RegI32 rhs, + Label* l) { + masm.branch32(c, lhs, rhs, l); +} + +void BaseCompiler::branchTo(Assembler::Condition c, RegI32 lhs, Imm32 rhs, + Label* l) { + masm.branch32(c, lhs, rhs, l); +} + +void BaseCompiler::branchTo(Assembler::Condition c, RegI64 lhs, RegI64 rhs, + Label* l) { + masm.branch64(c, lhs, rhs, l); +} + +void BaseCompiler::branchTo(Assembler::Condition c, RegI64 lhs, Imm64 rhs, + Label* l) { + masm.branch64(c, lhs, rhs, l); +} + +void BaseCompiler::branchTo(Assembler::Condition c, RegRef lhs, ImmWord rhs, + Label* l) { + masm.branchPtr(c, lhs, rhs, l); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Templated emitters + +template <> +inline BaseCompiler& BaseCompiler::selectCompiler() { + return *this; +} + +template <> +inline MacroAssembler& BaseCompiler::selectCompiler() { + return masm; +} + +template +void BaseCompiler::emitUnop(void (*op)(MacroAssembler& masm, SourceType rs, + DestType rd)) { + SourceType rs = pop(); + DestType rd = need(); + op(masm, rs, rd); + free(rs); + push(rd); +} + +// Specialize narrowing reuse. Consumers may assume that rs.reg==rd on 64-bit +// platforms, or rs.low==rd on 32-bit platforms. +template <> +inline void BaseCompiler::emitUnop(void (*op)(MacroAssembler& masm, RegI64 rs, + RegI32 rd)) { + RegI64 rs = pop(); + RegI32 rd = fromI64(rs); + op(masm, rs, rd); + freeI64Except(rs, rd); + push(rd); +} + +template +void BaseCompiler::emitUnop(void (*op)(CompilerType& compiler, RegType rsd)) { + RegType rsd = pop(); + op(selectCompiler(), rsd); + push(rsd); +} + +template +void BaseCompiler::emitUnop(void (*op)(BaseCompiler& bc, RegType rsd, + TempType rt), + TempType (*getSpecializedTemp)(BaseCompiler& bc)) { + RegType rsd = pop(); + TempType temp = getSpecializedTemp(*this); + op(*this, rsd, temp); + maybeFree(temp); + push(rsd); +} + +template +void BaseCompiler::emitUnop(void (*op)(MacroAssembler& masm, SourceType rs, + DestType rd, TempType temp)) { + SourceType rs = pop(); + DestType rd = need(); + TempType temp = need(); + op(masm, rs, rd, temp); + free(rs); + free(temp); + push(rd); +} + +template +void BaseCompiler::emitUnop(ImmType immediate, + void (*op)(MacroAssembler&, ImmType, SourceType, + DestType)) { + SourceType rs = pop(); + DestType rd = need(); + op(masm, immediate, rs, rd); + free(rs); + push(rd); +} + +template +void BaseCompiler::emitBinop(void (*op)(CompilerType& masm, RhsType src, + LhsDestType srcDest)) { + RhsType rs = pop(); + LhsDestType rsd = pop(); + op(selectCompiler(), rs, rsd); + free(rs); + push(rsd); +} + +template +void BaseCompiler::emitTernary(void (*op)(CompilerType&, ValType src0, + ValType src1, ValType srcDest)) { + ValType src2 = pop(); + ValType src1 = pop(); + ValType srcDest = pop(); + op(selectCompiler(), src1, src2, srcDest); + free(src2); + free(src1); + push(srcDest); +} + +template +void BaseCompiler::emitTernary(void (*op)(CompilerType&, ValType src0, + ValType src1, ValType srcDest, + ValType temp)) { + ValType src2 = pop(); + ValType src1 = pop(); + ValType srcDest = pop(); + ValType temp = need(); + op(selectCompiler(), src1, src2, srcDest, temp); + free(temp); + free(src2); + free(src1); + push(srcDest); +} + +template +void BaseCompiler::emitTernaryResultLast(void (*op)(CompilerType&, ValType src0, + ValType src1, + ValType srcDest)) { + ValType srcDest = pop(); + ValType src2 = pop(); + ValType src1 = pop(); + op(selectCompiler(), src1, src2, srcDest); + free(src2); + free(src1); + push(srcDest); +} + +template +void BaseCompiler::emitBinop(void (*op)(MacroAssembler& masm, RhsDestType src, + LhsType srcDest, RhsDestOp)) { + RhsDestType rsd = pop(); + LhsType rs = pop(); + op(masm, rsd, rs, RhsDestOp::True); + free(rs); + push(rsd); +} + +template +void BaseCompiler::emitBinop(void (*op)(MacroAssembler& masm, RhsType rs, + LhsDestType rsd, TempType temp)) { + RhsType rs = pop(); + LhsDestType rsd = pop(); + TempType temp = need(); + op(masm, rs, rsd, temp); + free(rs); + free(temp); + push(rsd); +} + +template +void BaseCompiler::emitBinop(void (*op)(MacroAssembler& masm, RhsType rs, + LhsDestType rsd, TempType1 temp1, + TempType2 temp2)) { + RhsType rs = pop(); + LhsDestType rsd = pop(); + TempType1 temp1 = need(); + TempType2 temp2 = need(); + op(masm, rs, rsd, temp1, temp2); + free(rs); + free(temp1); + free(temp2); + push(rsd); +} + +template +void BaseCompiler::emitBinop(ImmType immediate, + void (*op)(MacroAssembler&, ImmType, RhsType, + LhsDestType)) { + RhsType rs = pop(); + LhsDestType rsd = pop(); + op(masm, immediate, rs, rsd); + free(rs); + push(rsd); +} + +template +void BaseCompiler::emitBinop(ImmType immediate, + void (*op)(MacroAssembler&, ImmType, RhsType, + LhsDestType, TempType1 temp1, + TempType2 temp2)) { + RhsType rs = pop(); + LhsDestType rsd = pop(); + TempType1 temp1 = need(); + TempType2 temp2 = need(); + op(masm, immediate, rs, rsd, temp1, temp2); + free(rs); + free(temp1); + free(temp2); + push(rsd); +} + +template +void BaseCompiler::emitBinop(void (*op)(CompilerType1& compiler, RegType rs, + RegType rsd), + void (*opConst)(CompilerType2& compiler, ImmType c, + RegType rsd), + RegType (BaseCompiler::*rhsPopper)()) { + ImmType c; + if (popConst(&c)) { + RegType rsd = pop(); + opConst(selectCompiler(), c, rsd); + push(rsd); + } else { + RegType rs = rhsPopper ? (this->*rhsPopper)() : pop(); + RegType rsd = pop(); + op(selectCompiler(), rs, rsd); + free(rs); + push(rsd); + } +} + +template +bool BaseCompiler::emitInstanceCallOp(const SymbolicAddressSignature& fn, + R reader) { + if (!reader()) { + return false; + } + if (deadCode_) { + return true; + } + return emitInstanceCall(fn); +} + +template +bool BaseCompiler::emitInstanceCallOp(const SymbolicAddressSignature& fn, + R reader) { + A1 arg = 0; + if (!reader(&arg)) { + return false; + } + if (deadCode_) { + return true; + } + push(arg); + return emitInstanceCall(fn); +} + +template +bool BaseCompiler::emitInstanceCallOp(const SymbolicAddressSignature& fn, + R reader) { + A1 arg1 = 0; + A2 arg2 = 0; + if (!reader(&arg1, &arg2)) { + return false; + } + if (deadCode_) { + return true; + } + // Note order of arguments must be the same as for the reader. + push(arg1); + push(arg2); + return emitInstanceCall(fn); +} + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_codegen_inl_h diff --git a/js/src/wasm/WasmBCDefs.h b/js/src/wasm/WasmBCDefs.h new file mode 100644 index 0000000000..d3bd547460 --- /dev/null +++ b/js/src/wasm/WasmBCDefs.h @@ -0,0 +1,201 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: common configuration +// and simple definitions; all include directives. + +#ifndef wasm_wasm_baseline_defs_h +#define wasm_wasm_baseline_defs_h + +#include "mozilla/MathAlgorithms.h" +#include "mozilla/Maybe.h" +#include "mozilla/ScopeExit.h" + +#include +#include + +#include "jit/AtomicOp.h" +#include "jit/IonTypes.h" +#include "jit/JitAllocPolicy.h" +#include "jit/Label.h" +#include "jit/RegisterAllocator.h" +#include "jit/Registers.h" +#include "jit/RegisterSets.h" +#if defined(JS_CODEGEN_ARM) +# include "jit/arm/Assembler-arm.h" +#endif +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) +# include "jit/x86-shared/Architecture-x86-shared.h" +# include "jit/x86-shared/Assembler-x86-shared.h" +#endif +#if defined(JS_CODEGEN_MIPS64) +# include "jit/mips-shared/Assembler-mips-shared.h" +# include "jit/mips64/Assembler-mips64.h" +#endif +#if defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/Assembler-loong64.h" +#endif +#if defined(JS_CODEGEN_RISCV64) +# include "jit/riscv64/Assembler-riscv64.h" +#endif +#include "js/ScalarType.h" +#include "util/Memory.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmDebugFrame.h" +#include "wasm/WasmGC.h" +#include "wasm/WasmGcObject.h" +#include "wasm/WasmGenerator.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmOpIter.h" +#include "wasm/WasmSignalHandlers.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmValidate.h" + +using mozilla::DebugOnly; +using mozilla::FloorLog2; +using mozilla::IsPowerOfTwo; +using mozilla::Maybe; +using mozilla::Nothing; +using mozilla::Some; + +namespace js { +namespace wasm { + +using HandleNaNSpecially = bool; +using InvertBranch = bool; +using IsKnownNotZero = bool; +using IsUnsigned = bool; +using IsRemainder = bool; +using NeedsBoundsCheck = bool; +using WantResult = bool; +using ZeroOnOverflow = bool; + +class BaseStackFrame; + +// Two flags, useABI and restoreRegisterStateAndRealm, control how calls are +// made. +// +// UseABI::Wasm implies that the Instance/Heap/Global registers are nonvolatile, +// except when RestoreRegisterStateAndRealm::True is also set, when they are +// volatile. +// +// UseABI::Builtin implies that the Instance/Heap/Global registers are volatile. +// In this case, we require RestoreRegisterStateAndRealm::False. The calling +// convention is otherwise like UseABI::Wasm. +// +// UseABI::System implies that the Instance/Heap/Global registers are volatile. +// Additionally, the parameter passing mechanism may be slightly different from +// the UseABI::Wasm convention. +// +// When the Instance/Heap/Global registers are not volatile, the baseline +// compiler will restore the Instance register from its save slot before the +// call, since the baseline compiler uses the Instance register for other +// things. +// +// When those registers are volatile, the baseline compiler will reload them +// after the call (it will restore the Instance register from the save slot and +// load the other two from the Instance data). + +enum class UseABI { Wasm, Builtin, System }; +enum class RestoreRegisterStateAndRealm { False = false, True = true }; +enum class RhsDestOp { True = true }; + +// Compiler configuration. +// +// The following internal configuration #defines are used. The configuration is +// partly below in this file, partly in WasmBCRegDefs.h. +// +// RABALDR_PIN_INSTANCE +// InstanceReg is not allocatable and always holds the current Instance*, +// except in known contexts where it could have been clobbered, such as after +// certain calls. +// +// RABALDR_ZERO_EXTENDS +// The canonical representation of a 32-bit value in a 64-bit register is +// zero-extended. For 64-bit platforms only. See comment block "64-bit GPRs +// carrying 32-bit values" in MacroAssembler.h. +// +// RABALDR_CHUNKY_STACK +// The platform must allocate the CPU stack in chunks and not word-at-a-time +// due to SP alignment requirements (ARM64 for now). +// +// RABALDR_INT_DIV_I64_CALLOUT +// The platform calls out to the runtime to divide i64/u64. +// +// RABALDR_I64_TO_FLOAT_CALLOUT +// The platform calls out to the runtime for i64 -> fXX conversions. +// +// RABALDR_FLOAT_TO_I64_CALLOUT +// The platform calls out to the runtime for fXX -> i64 conversions. +// +// RABALDR_SCRATCH_ +// The baseline compiler has its own scratch registers for the given type, it +// does not use the MacroAssembler's scratch. This is really an anachronism - +// the baseline compiler should never use the MacroAssembler's scratches. +// +// RABALDR_SCRATCH_F32_ALIASES_F64 +// On a platform where the baseline compiler has its own F32 and F64 +// scratches, these are the same register. + +#ifdef JS_CODEGEN_X64 +# define RABALDR_ZERO_EXTENDS +# define RABALDR_PIN_INSTANCE +#endif + +#ifdef JS_CODEGEN_ARM64 +# define RABALDR_CHUNKY_STACK +# define RABALDR_ZERO_EXTENDS +# define RABALDR_PIN_INSTANCE +#endif + +#ifdef JS_CODEGEN_X86 +# define RABALDR_INT_DIV_I64_CALLOUT +#endif + +#ifdef JS_CODEGEN_ARM +# define RABALDR_INT_DIV_I64_CALLOUT +# define RABALDR_I64_TO_FLOAT_CALLOUT +# define RABALDR_FLOAT_TO_I64_CALLOUT +#endif + +#ifdef JS_CODEGEN_MIPS64 +# define RABALDR_PIN_INSTANCE +#endif + +#ifdef JS_CODEGEN_LOONG64 +# define RABALDR_PIN_INSTANCE +#endif + +#ifdef JS_CODEGEN_RISCV64 +# define RABALDR_PIN_INSTANCE +#endif + +// Max number of pushes onto the value stack for any opcode or emitter that +// does not push a variable, unbounded amount (anything with multiple +// results). This includes also intermediate pushes such as values pushed as +// parameters for builtin calls. +// +// This limit is set quite high on purpose, so as to avoid brittleness. The +// true max value is likely no more than four or five. + +static constexpr size_t MaxPushesPerOpcode = 10; + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_defs_h diff --git a/js/src/wasm/WasmBCFrame.cpp b/js/src/wasm/WasmBCFrame.cpp new file mode 100644 index 0000000000..454f732823 --- /dev/null +++ b/js/src/wasm/WasmBCFrame.cpp @@ -0,0 +1,544 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmBCFrame.h" + +#include "wasm/WasmBaselineCompile.h" // For BaseLocalIter +#include "wasm/WasmBCClass.h" + +#include "jit/MacroAssembler-inl.h" +#include "wasm/WasmBCClass-inl.h" +#include "wasm/WasmBCCodegen-inl.h" +#include "wasm/WasmBCRegDefs-inl.h" +#include "wasm/WasmBCRegMgmt-inl.h" +#include "wasm/WasmBCStkMgmt-inl.h" + +namespace js { +namespace wasm { + +////////////////////////////////////////////////////////////////////////////// +// +// BaseLocalIter methods. + +BaseLocalIter::BaseLocalIter(const ValTypeVector& locals, + const ArgTypeVector& args, bool debugEnabled) + : locals_(locals), + args_(args), + argsIter_(args_), + index_(0), + frameSize_(0), + nextFrameSize_(debugEnabled ? DebugFrame::offsetOfFrame() : 0), + frameOffset_(INT32_MAX), + stackResultPointerOffset_(INT32_MAX), + mirType_(MIRType::Undefined), + done_(false) { + MOZ_ASSERT(args.lengthWithoutStackResults() <= locals.length()); + settle(); +} + +int32_t BaseLocalIter::pushLocal(size_t nbytes) { + MOZ_ASSERT(nbytes % 4 == 0 && nbytes <= 16); + nextFrameSize_ = AlignBytes(frameSize_, nbytes) + nbytes; + return nextFrameSize_; // Locals grow down so capture base address. +} + +void BaseLocalIter::settle() { + MOZ_ASSERT(!done_); + frameSize_ = nextFrameSize_; + + if (!argsIter_.done()) { + mirType_ = argsIter_.mirType(); + MIRType concreteType = mirType_; + switch (mirType_) { + case MIRType::StackResults: + // The pointer to stack results is handled like any other argument: + // either addressed in place if it is passed on the stack, or we spill + // it in the frame if it's in a register. + MOZ_ASSERT(args_.isSyntheticStackResultPointerArg(index_)); + concreteType = MIRType::Pointer; + [[fallthrough]]; + case MIRType::Int32: + case MIRType::Int64: + case MIRType::Double: + case MIRType::Float32: + case MIRType::RefOrNull: +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: +#endif + if (argsIter_->argInRegister()) { + frameOffset_ = pushLocal(MIRTypeToSize(concreteType)); + } else { + frameOffset_ = -(argsIter_->offsetFromArgBase() + sizeof(Frame)); + } + break; + default: + MOZ_CRASH("Argument type"); + } + if (mirType_ == MIRType::StackResults) { + stackResultPointerOffset_ = frameOffset(); + // Advance past the synthetic stack result pointer argument and fall + // through to the next case. + argsIter_++; + frameSize_ = nextFrameSize_; + MOZ_ASSERT(argsIter_.done()); + } else { + return; + } + } + + if (index_ < locals_.length()) { + switch (locals_[index_].kind()) { + case ValType::I32: + case ValType::I64: + case ValType::F32: + case ValType::F64: +#ifdef ENABLE_WASM_SIMD + case ValType::V128: +#endif + case ValType::Ref: + // TODO/AnyRef-boxing: With boxed immediates and strings, the + // debugger must be made aware that AnyRef != Pointer. + ASSERT_ANYREF_IS_JSOBJECT; + mirType_ = locals_[index_].toMIRType(); + frameOffset_ = pushLocal(MIRTypeToSize(mirType_)); + break; + default: + MOZ_CRASH("Compiler bug: Unexpected local type"); + } + return; + } + + done_ = true; +} + +void BaseLocalIter::operator++(int) { + MOZ_ASSERT(!done_); + index_++; + if (!argsIter_.done()) { + argsIter_++; + } + settle(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Stack map methods. + +bool BaseCompiler::createStackMap(const char* who) { + const ExitStubMapVector noExtras; + return stackMapGenerator_.createStackMap(who, noExtras, masm.currentOffset(), + HasDebugFrameWithLiveRefs::No, stk_); +} + +bool BaseCompiler::createStackMap(const char* who, CodeOffset assemblerOffset) { + const ExitStubMapVector noExtras; + return stackMapGenerator_.createStackMap(who, noExtras, + assemblerOffset.offset(), + HasDebugFrameWithLiveRefs::No, stk_); +} + +bool BaseCompiler::createStackMap( + const char* who, HasDebugFrameWithLiveRefs debugFrameWithLiveRefs) { + const ExitStubMapVector noExtras; + return stackMapGenerator_.createStackMap(who, noExtras, masm.currentOffset(), + debugFrameWithLiveRefs, stk_); +} + +bool BaseCompiler::createStackMap( + const char* who, const ExitStubMapVector& extras, uint32_t assemblerOffset, + HasDebugFrameWithLiveRefs debugFrameWithLiveRefs) { + return stackMapGenerator_.createStackMap(who, extras, assemblerOffset, + debugFrameWithLiveRefs, stk_); +} + +bool MachineStackTracker::cloneTo(MachineStackTracker* dst) { + MOZ_ASSERT(dst->vec_.empty()); + if (!dst->vec_.appendAll(vec_)) { + return false; + } + dst->numPtrs_ = numPtrs_; + return true; +} + +bool StackMapGenerator::generateStackmapEntriesForTrapExit( + const ArgTypeVector& args, ExitStubMapVector* extras) { + return GenerateStackmapEntriesForTrapExit(args, trapExitLayout_, + trapExitLayoutNumWords_, extras); +} + +bool StackMapGenerator::createStackMap( + const char* who, const ExitStubMapVector& extras, uint32_t assemblerOffset, + HasDebugFrameWithLiveRefs debugFrameWithLiveRefs, const StkVector& stk) { + size_t countedPointers = machineStackTracker.numPtrs() + memRefsOnStk; +#ifndef DEBUG + // An important optimization. If there are obviously no pointers, as + // we expect in the majority of cases, exit quickly. + if (countedPointers == 0 && + debugFrameWithLiveRefs == HasDebugFrameWithLiveRefs::No) { + // We can skip creating the map if there are no |true| elements in + // |extras|. + bool extrasHasRef = false; + for (bool b : extras) { + if (b) { + extrasHasRef = true; + break; + } + } + if (!extrasHasRef) { + return true; + } + } +#else + // In the debug case, create the stackmap regardless, and cross-check + // the pointer-counting below. We expect the final map to have + // |countedPointers| in total. This doesn't include those in the + // DebugFrame, but they do not appear in the map's bitmap. Note that + // |countedPointers| is debug-only from this point onwards. + for (bool b : extras) { + countedPointers += (b ? 1 : 0); + } +#endif + + // Start with the frame-setup map, and add operand-stack information to + // that. augmentedMst holds live data only within individual calls to + // createStackMap. + augmentedMst.clear(); + if (!machineStackTracker.cloneTo(&augmentedMst)) { + return false; + } + + // At this point, augmentedMst only contains entries covering the + // incoming argument area (if any) and for the area allocated by this + // function's prologue. We now need to calculate how far the machine's + // stack pointer is below where it was at the start of the body. But we + // must take care not to include any words pushed as arguments to an + // upcoming function call, since those words "belong" to the stackmap of + // the callee, not to the stackmap of this function. Note however that + // any alignment padding pushed prior to pushing the args *does* belong to + // this function. + // + // That padding is taken into account at the point where + // framePushedExcludingOutboundCallArgs is set, viz, in startCallArgs(), + // and comprises two components: + // + // * call->frameAlignAdjustment + // * the padding applied to the stack arg area itself. That is: + // StackArgAreaSize(argTys) - StackArgAreaSizeUnpadded(argTys) + Maybe framePushedExcludingArgs; + if (framePushedAtEntryToBody.isNothing()) { + // Still in the prologue. framePushedExcludingArgs remains Nothing. + MOZ_ASSERT(framePushedExcludingOutboundCallArgs.isNothing()); + } else { + // In the body. + MOZ_ASSERT(masm_.framePushed() >= framePushedAtEntryToBody.value()); + if (framePushedExcludingOutboundCallArgs.isSome()) { + // In the body, and we've potentially pushed some args onto the stack. + // We must ignore them when sizing the stackmap. + MOZ_ASSERT(masm_.framePushed() >= + framePushedExcludingOutboundCallArgs.value()); + MOZ_ASSERT(framePushedExcludingOutboundCallArgs.value() >= + framePushedAtEntryToBody.value()); + framePushedExcludingArgs = + Some(framePushedExcludingOutboundCallArgs.value()); + } else { + // In the body, but not with call args on the stack. The stackmap + // must be sized so as to extend all the way "down" to + // masm_.framePushed(). + framePushedExcludingArgs = Some(masm_.framePushed()); + } + } + + if (framePushedExcludingArgs.isSome()) { + uint32_t bodyPushedBytes = + framePushedExcludingArgs.value() - framePushedAtEntryToBody.value(); + MOZ_ASSERT(0 == bodyPushedBytes % sizeof(void*)); + if (!augmentedMst.pushNonGCPointers(bodyPushedBytes / sizeof(void*))) { + return false; + } + } + + // Scan the operand stack, marking pointers in the just-added new + // section. + MOZ_ASSERT_IF(framePushedAtEntryToBody.isNothing(), stk.empty()); + MOZ_ASSERT_IF(framePushedExcludingArgs.isNothing(), stk.empty()); + + for (const Stk& v : stk) { +#ifndef DEBUG + // We don't track roots in registers, per rationale below, so if this + // doesn't hold, something is seriously wrong, and we're likely to get a + // GC-related crash. + MOZ_RELEASE_ASSERT(v.kind() != Stk::RegisterRef); + if (v.kind() != Stk::MemRef) { + continue; + } +#else + // Take the opportunity to check everything we reasonably can about + // operand stack elements. + switch (v.kind()) { + case Stk::MemI32: + case Stk::MemI64: + case Stk::MemF32: + case Stk::MemF64: + case Stk::ConstI32: + case Stk::ConstI64: + case Stk::ConstF32: + case Stk::ConstF64: +# ifdef ENABLE_WASM_SIMD + case Stk::MemV128: + case Stk::ConstV128: +# endif + // All of these have uninteresting type. + continue; + case Stk::LocalI32: + case Stk::LocalI64: + case Stk::LocalF32: + case Stk::LocalF64: +# ifdef ENABLE_WASM_SIMD + case Stk::LocalV128: +# endif + // These also have uninteresting type. Check that they live in the + // section of stack set up by beginFunction(). The unguarded use of + // |value()| here is safe due to the assertion above this loop. + MOZ_ASSERT(v.offs() <= framePushedAtEntryToBody.value()); + continue; + case Stk::RegisterI32: + case Stk::RegisterI64: + case Stk::RegisterF32: + case Stk::RegisterF64: +# ifdef ENABLE_WASM_SIMD + case Stk::RegisterV128: +# endif + // These also have uninteresting type, but more to the point: all + // registers holding live values should have been flushed to the + // machine stack immediately prior to the instruction to which this + // stackmap pertains. So these can't happen. + MOZ_CRASH("createStackMap: operand stack has Register-non-Ref"); + case Stk::MemRef: + // This is the only case we care about. We'll handle it after the + // switch. + break; + case Stk::LocalRef: + // We need the stackmap to mention this pointer, but it should + // already be in the machineStackTracker section created by + // beginFunction(). + MOZ_ASSERT(v.offs() <= framePushedAtEntryToBody.value()); + continue; + case Stk::ConstRef: + // This can currently only be a null pointer. + MOZ_ASSERT(v.refval() == 0); + continue; + case Stk::RegisterRef: + // This can't happen, per rationale above. + MOZ_CRASH("createStackMap: operand stack contains RegisterRef"); + default: + MOZ_CRASH("createStackMap: unknown operand stack element"); + } +#endif + // v.offs() holds masm.framePushed() at the point immediately after it + // was pushed on the stack. Since it's still on the stack, + // masm.framePushed() can't be less. + MOZ_ASSERT(v.offs() <= framePushedExcludingArgs.value()); + uint32_t offsFromMapLowest = framePushedExcludingArgs.value() - v.offs(); + MOZ_ASSERT(0 == offsFromMapLowest % sizeof(void*)); + augmentedMst.setGCPointer(offsFromMapLowest / sizeof(void*)); + } + + // Create the final StackMap. The initial map is zeroed out, so there's + // no need to write zero bits in it. + const uint32_t extraWords = extras.length(); + const uint32_t augmentedMstWords = augmentedMst.length(); + const uint32_t numMappedWords = extraWords + augmentedMstWords; + StackMap* stackMap = StackMap::create(numMappedWords); + if (!stackMap) { + return false; + } + + { + // First the exit stub extra words, if any. + uint32_t i = 0; + for (bool b : extras) { + if (b) { + stackMap->setBit(i); + } + i++; + } + } + { + // Followed by the "main" part of the map. + // + // This is really just a bit-array copy, so it is reasonable to ask + // whether the representation of MachineStackTracker could be made more + // similar to that of StackMap, so that the copy could be done with + // `memcpy`. Unfortunately it's not so simple; see comment on `class + // MachineStackTracker` for details. + MachineStackTracker::Iter iter(augmentedMst); + while (true) { + size_t i = iter.get(); + if (i == MachineStackTracker::Iter::FINISHED) { + break; + } + stackMap->setBit(extraWords + i); + } + } + + stackMap->setExitStubWords(extraWords); + + // Record in the map, how far down from the highest address the Frame* is. + // Take the opportunity to check that we haven't marked any part of the + // Frame itself as a pointer. + stackMap->setFrameOffsetFromTop(numStackArgWords + + sizeof(Frame) / sizeof(void*)); +#ifdef DEBUG + for (uint32_t i = 0; i < sizeof(Frame) / sizeof(void*); i++) { + MOZ_ASSERT(stackMap->getBit(stackMap->header.numMappedWords - + stackMap->header.frameOffsetFromTop + i) == 0); + } +#endif + + // Note the presence of a DebugFrame with live pointers, if any. + if (debugFrameWithLiveRefs != HasDebugFrameWithLiveRefs::No) { + stackMap->setHasDebugFrameWithLiveRefs(); + } + + // Add the completed map to the running collection thereof. + if (!stackMaps_->add((uint8_t*)(uintptr_t)assemblerOffset, stackMap)) { + stackMap->destroy(); + return false; + } + +#ifdef DEBUG + { + // Crosscheck the map pointer counting. + uint32_t nw = stackMap->header.numMappedWords; + uint32_t np = 0; + for (uint32_t i = 0; i < nw; i++) { + np += stackMap->getBit(i); + } + MOZ_ASSERT(size_t(np) == countedPointers); + } +#endif + + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Stack frame methods. + +void BaseStackFrame::zeroLocals(BaseRegAlloc* ra) { + MOZ_ASSERT(varLow_ != UINT32_MAX); + + if (varLow_ == varHigh_) { + return; + } + + static const uint32_t wordSize = sizeof(void*); + + // The adjustments to 'low' by the size of the item being stored compensates + // for the fact that locals offsets are the offsets from Frame to the bytes + // directly "above" the locals in the locals area. See comment at Local. + + // On 64-bit systems we may have 32-bit alignment for the local area as it + // may be preceded by parameters and prologue/debug data. + + uint32_t low = varLow_; + if (low % wordSize) { + masm.store32(Imm32(0), Address(sp_, localOffset(low + 4))); + low += 4; + } + MOZ_ASSERT(low % wordSize == 0); + + const uint32_t high = AlignBytes(varHigh_, wordSize); + + // An UNROLL_LIMIT of 16 is chosen so that we only need an 8-bit signed + // immediate to represent the offset in the store instructions in the loop + // on x64. + + const uint32_t UNROLL_LIMIT = 16; + const uint32_t initWords = (high - low) / wordSize; + const uint32_t tailWords = initWords % UNROLL_LIMIT; + const uint32_t loopHigh = high - (tailWords * wordSize); + + // With only one word to initialize, just store an immediate zero. + + if (initWords == 1) { + masm.storePtr(ImmWord(0), Address(sp_, localOffset(low + wordSize))); + return; + } + + // For other cases, it's best to have a zero in a register. + // + // One can do more here with SIMD registers (store 16 bytes at a time) or + // with instructions like STRD on ARM (store 8 bytes at a time), but that's + // for another day. + + RegI32 zero = ra->needI32(); + masm.mov(ImmWord(0), zero); + + // For the general case we want to have a loop body of UNROLL_LIMIT stores + // and then a tail of less than UNROLL_LIMIT stores. When initWords is less + // than 2*UNROLL_LIMIT the loop trip count is at most 1 and there is no + // benefit to having the pointer calculations and the compare-and-branch. + // So we completely unroll when we have initWords < 2 * UNROLL_LIMIT. (In + // this case we'll end up using 32-bit offsets on x64 for up to half of the + // stores, though.) + + // Fully-unrolled case. + + if (initWords < 2 * UNROLL_LIMIT) { + for (uint32_t i = low; i < high; i += wordSize) { + masm.storePtr(zero, Address(sp_, localOffset(i + wordSize))); + } + ra->freeI32(zero); + return; + } + + // Unrolled loop with a tail. Stores will use negative offsets. That's OK + // for x86 and ARM, at least. + + // Compute pointer to the highest-addressed slot on the frame. + RegI32 p = ra->needI32(); + masm.computeEffectiveAddress(Address(sp_, localOffset(low + wordSize)), p); + + // Compute pointer to the lowest-addressed slot on the frame that will be + // initialized by the loop body. + RegI32 lim = ra->needI32(); + masm.computeEffectiveAddress(Address(sp_, localOffset(loopHigh + wordSize)), + lim); + + // The loop body. Eventually we'll have p == lim and exit the loop. + Label again; + masm.bind(&again); + for (uint32_t i = 0; i < UNROLL_LIMIT; ++i) { + masm.storePtr(zero, Address(p, -(wordSize * i))); + } + masm.subPtr(Imm32(UNROLL_LIMIT * wordSize), p); + masm.branchPtr(Assembler::LessThan, lim, p, &again); + + // The tail. + for (uint32_t i = 0; i < tailWords; ++i) { + masm.storePtr(zero, Address(p, -(wordSize * i))); + } + + ra->freeI32(p); + ra->freeI32(lim); + ra->freeI32(zero); +} + +} // namespace wasm +} // namespace js diff --git a/js/src/wasm/WasmBCFrame.h b/js/src/wasm/WasmBCFrame.h new file mode 100644 index 0000000000..089d3d5e84 --- /dev/null +++ b/js/src/wasm/WasmBCFrame.h @@ -0,0 +1,1397 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: CPU stack frame, +// stack maps, and associated logic. + +#ifndef wasm_wasm_baseline_frame_h +#define wasm_wasm_baseline_frame_h + +#include "wasm/WasmBaselineCompile.h" // For BaseLocalIter +#include "wasm/WasmBCDefs.h" +#include "wasm/WasmBCRegDefs.h" +#include "wasm/WasmBCStk.h" +#include "wasm/WasmConstants.h" // For MaxFrameSize + +// [SMDOC] Wasm baseline compiler's stack frame. +// +// For background, see "Wasm's ABIs" in WasmFrame.h, the following should never +// be in conflict with that. +// +// The stack frame has four parts ("below" means at lower addresses): +// +// - the Frame element; +// - the Local area, including the DebugFrame element and possibly a spilled +// pointer to stack results, if any; allocated below the header with various +// forms of alignment; +// - the Dynamic area, comprising the temporary storage the compiler uses for +// register spilling, allocated below the Local area; +// - the Arguments area, comprising memory allocated for outgoing calls, +// allocated below the Dynamic area. +// +// +==============================+ +// | Incoming stack arg | +// | ... | +// ------------- +==============================+ +// | Frame (fixed size) | +// ------------- +==============================+ <-------------------- FP +// ^ | DebugFrame (optional) | ^ ^ ^^ +// localSize | Register arg local | | | || +// | | ... | | | framePushed +// | | Register stack result ptr?| | | || +// | | Non-arg local | | | || +// | | ... | | | || +// | | (padding) | | | || +// | | Instance pointer | | | || +// | +------------------------------+ | | || +// v | (padding) | | v || +// ------------- +==============================+ currentStackHeight || +// ^ | Dynamic (variable size) | | || +// dynamicSize | ... | | || +// v | ... | v || +// ------------- | (free space, sometimes) | --------- v| +// +==============================+ <----- SP not-during calls +// | Arguments (sometimes) | | +// | ... | v +// +==============================+ <----- SP during calls +// +// The Frame is addressed off the stack pointer. masm.framePushed() is always +// correct, and masm.getStackPointer() + masm.framePushed() always addresses the +// Frame, with the DebugFrame optionally below it. +// +// The Local area (including the DebugFrame and, if needed, the spilled value of +// the stack results area pointer) is laid out by BaseLocalIter and is allocated +// and deallocated by standard prologue and epilogue functions that manipulate +// the stack pointer, but it is accessed via BaseStackFrame. +// +// The Dynamic area is maintained by and accessed via BaseStackFrame. On some +// systems (such as ARM64), the Dynamic memory may be allocated in chunks +// because the SP needs a specific alignment, and in this case there will +// normally be some free space directly above the SP. The stack height does not +// include the free space, it reflects the logically used space only. +// +// The Dynamic area is where space for stack results is allocated when calling +// functions that return results on the stack. If a function has stack results, +// a pointer to the low address of the stack result area is passed as an +// additional argument, according to the usual ABI. See +// ABIResultIter::HasStackResults. +// +// The Arguments area is allocated and deallocated via BaseStackFrame (see +// comments later) but is accessed directly off the stack pointer. + +namespace js { +namespace wasm { + +using namespace js::jit; + +// Abstraction of the height of the stack frame, to avoid type confusion. + +class StackHeight { + friend class BaseStackFrameAllocator; + + uint32_t height; + + public: + explicit StackHeight(uint32_t h) : height(h) {} + static StackHeight Invalid() { return StackHeight(UINT32_MAX); } + bool isValid() const { return height != UINT32_MAX; } + bool operator==(StackHeight rhs) const { + MOZ_ASSERT(isValid() && rhs.isValid()); + return height == rhs.height; + } + bool operator!=(StackHeight rhs) const { return !(*this == rhs); } +}; + +// Abstraction for where multi-value results go on the machine stack. + +class StackResultsLoc { + uint32_t bytes_; + size_t count_; + Maybe height_; + + public: + StackResultsLoc() : bytes_(0), count_(0){}; + StackResultsLoc(uint32_t bytes, size_t count, uint32_t height) + : bytes_(bytes), count_(count), height_(Some(height)) { + MOZ_ASSERT(bytes != 0); + MOZ_ASSERT(count != 0); + MOZ_ASSERT(height != 0); + } + + uint32_t bytes() const { return bytes_; } + uint32_t count() const { return count_; } + uint32_t height() const { return height_.value(); } + + bool hasStackResults() const { return bytes() != 0; } + StackResults stackResults() const { + return hasStackResults() ? StackResults::HasStackResults + : StackResults::NoStackResults; + } +}; + +// Abstraction of the baseline compiler's stack frame (except for the Frame / +// DebugFrame parts). See comments above for more. Remember, "below" on the +// stack means at lower addresses. +// +// The abstraction is split into two parts: BaseStackFrameAllocator is +// responsible for allocating and deallocating space on the stack and for +// performing computations that are affected by how the allocation is performed; +// BaseStackFrame then provides a pleasant interface for stack frame management. + +class BaseStackFrameAllocator { + MacroAssembler& masm; + +#ifdef RABALDR_CHUNKY_STACK + // On platforms that require the stack pointer to be aligned on a boundary + // greater than the typical stack item (eg, ARM64 requires 16-byte alignment + // but items are 8 bytes), allocate stack memory in chunks, and use a + // separate stack height variable to track the effective stack pointer + // within the allocated area. Effectively, there's a variable amount of + // free space directly above the stack pointer. See diagram above. + + // The following must be true in order for the stack height to be + // predictable at control flow joins: + // + // - The Local area is always aligned according to WasmStackAlignment, ie, + // masm.framePushed() % WasmStackAlignment is zero after allocating + // locals. + // + // - ChunkSize is always a multiple of WasmStackAlignment. + // + // - Pushing and popping are always in units of ChunkSize (hence preserving + // alignment). + // + // - The free space on the stack (masm.framePushed() - currentStackHeight_) + // is a predictable (nonnegative) amount. + + // As an optimization, we pre-allocate some space on the stack, the size of + // this allocation is InitialChunk and it must be a multiple of ChunkSize. + // It is allocated as part of the function prologue and deallocated as part + // of the epilogue, along with the locals. + // + // If ChunkSize is too large then we risk overflowing the stack on simple + // recursions with few live values where stack overflow should not be a + // risk; if it is too small we spend too much time adjusting the stack + // pointer. + // + // Good values for ChunkSize are the subject of future empirical analysis; + // eight words is just an educated guess. + + static constexpr uint32_t ChunkSize = 8 * sizeof(void*); + static constexpr uint32_t InitialChunk = ChunkSize; + + // The current logical height of the frame is + // currentStackHeight_ = localSize_ + dynamicSize + // where dynamicSize is not accounted for explicitly and localSize_ also + // includes size for the DebugFrame. + // + // The allocated size of the frame, provided by masm.framePushed(), is usually + // larger than currentStackHeight_, notably at the beginning of execution when + // we've allocated InitialChunk extra space. + + uint32_t currentStackHeight_; +#endif + + // Size of the Local area in bytes (stable after BaseCompiler::init() has + // called BaseStackFrame::setupLocals(), which in turn calls + // BaseStackFrameAllocator::setLocalSize()), always rounded to the proper + // stack alignment. The Local area is then allocated in beginFunction(), + // following the allocation of the Header. See onFixedStackAllocated() + // below. + + uint32_t localSize_; + + protected: + /////////////////////////////////////////////////////////////////////////// + // + // Initialization + + explicit BaseStackFrameAllocator(MacroAssembler& masm) + : masm(masm), +#ifdef RABALDR_CHUNKY_STACK + currentStackHeight_(0), +#endif + localSize_(UINT32_MAX) { + } + + protected: + ////////////////////////////////////////////////////////////////////// + // + // The Local area - the static part of the frame. + + // Record the size of the Local area, once it is known. + + void setLocalSize(uint32_t localSize) { + MOZ_ASSERT(localSize == AlignBytes(localSize, sizeof(void*)), + "localSize_ should be aligned to at least a pointer"); + MOZ_ASSERT(localSize_ == UINT32_MAX); + localSize_ = localSize; + } + + // Record the current stack height, after it has become stable in + // beginFunction(). See also BaseStackFrame::onFixedStackAllocated(). + + void onFixedStackAllocated() { + MOZ_ASSERT(localSize_ != UINT32_MAX); +#ifdef RABALDR_CHUNKY_STACK + currentStackHeight_ = localSize_; +#endif + } + + public: + // The fixed amount of memory, in bytes, allocated on the stack below the + // Header for purposes such as locals and other fixed values. Includes all + // necessary alignment, and on ARM64 also the initial chunk for the working + // stack memory. + + uint32_t fixedAllocSize() const { + MOZ_ASSERT(localSize_ != UINT32_MAX); +#ifdef RABALDR_CHUNKY_STACK + return localSize_ + InitialChunk; +#else + return localSize_; +#endif + } + +#ifdef RABALDR_CHUNKY_STACK + // The allocated frame size is frequently larger than the logical stack + // height; we round up to a chunk boundary, and special case the initial + // chunk. + uint32_t framePushedForHeight(uint32_t logicalHeight) { + if (logicalHeight <= fixedAllocSize()) { + return fixedAllocSize(); + } + return fixedAllocSize() + + AlignBytes(logicalHeight - fixedAllocSize(), ChunkSize); + } +#endif + + protected: + ////////////////////////////////////////////////////////////////////// + // + // The Dynamic area - the dynamic part of the frame, for spilling and saving + // intermediate values. + + // Offset off of sp_ for the slot at stack area location `offset`. + + int32_t stackOffset(int32_t offset) { + MOZ_ASSERT(offset > 0); + return masm.framePushed() - offset; + } + + uint32_t computeHeightWithStackResults(StackHeight stackBase, + uint32_t stackResultBytes) { + MOZ_ASSERT(stackResultBytes); + MOZ_ASSERT(currentStackHeight() >= stackBase.height); + return stackBase.height + stackResultBytes; + } + +#ifdef RABALDR_CHUNKY_STACK + void pushChunkyBytes(uint32_t bytes) { + checkChunkyInvariants(); + uint32_t freeSpace = masm.framePushed() - currentStackHeight_; + if (freeSpace < bytes) { + uint32_t bytesToReserve = AlignBytes(bytes - freeSpace, ChunkSize); + MOZ_ASSERT(bytesToReserve + freeSpace >= bytes); + masm.reserveStack(bytesToReserve); + } + currentStackHeight_ += bytes; + checkChunkyInvariants(); + } + + void popChunkyBytes(uint32_t bytes) { + checkChunkyInvariants(); + currentStackHeight_ -= bytes; + // Sometimes, popChunkyBytes() is used to pop a larger area, as when we drop + // values consumed by a call, and we may need to drop several chunks. But + // never drop the initial chunk. Crucially, the amount we drop is always an + // integral number of chunks. + uint32_t freeSpace = masm.framePushed() - currentStackHeight_; + if (freeSpace >= ChunkSize) { + uint32_t targetAllocSize = framePushedForHeight(currentStackHeight_); + uint32_t amountToFree = masm.framePushed() - targetAllocSize; + MOZ_ASSERT(amountToFree % ChunkSize == 0); + if (amountToFree) { + masm.freeStack(amountToFree); + } + } + checkChunkyInvariants(); + } +#endif + + uint32_t currentStackHeight() const { +#ifdef RABALDR_CHUNKY_STACK + return currentStackHeight_; +#else + return masm.framePushed(); +#endif + } + + private: +#ifdef RABALDR_CHUNKY_STACK + void checkChunkyInvariants() { + MOZ_ASSERT(masm.framePushed() >= fixedAllocSize()); + MOZ_ASSERT(masm.framePushed() >= currentStackHeight_); + MOZ_ASSERT(masm.framePushed() == fixedAllocSize() || + masm.framePushed() - currentStackHeight_ < ChunkSize); + MOZ_ASSERT((masm.framePushed() - localSize_) % ChunkSize == 0); + } +#endif + + // For a given stack height, return the appropriate size of the allocated + // frame. + + uint32_t framePushedForHeight(StackHeight stackHeight) { +#ifdef RABALDR_CHUNKY_STACK + // A more complicated adjustment is needed. + return framePushedForHeight(stackHeight.height); +#else + // The allocated frame size equals the stack height. + return stackHeight.height; +#endif + } + + public: + // The current height of the stack area, not necessarily zero-based, in a + // type-safe way. + + StackHeight stackHeight() const { return StackHeight(currentStackHeight()); } + + // Set the frame height to a previously recorded value. + + void setStackHeight(StackHeight amount) { +#ifdef RABALDR_CHUNKY_STACK + currentStackHeight_ = amount.height; + masm.setFramePushed(framePushedForHeight(amount)); + checkChunkyInvariants(); +#else + masm.setFramePushed(amount.height); +#endif + } + + // The current height of the dynamic part of the stack area (ie, the backing + // store for the evaluation stack), zero-based. + + uint32_t dynamicHeight() const { return currentStackHeight() - localSize_; } + + // Before branching to an outer control label, pop the execution stack to + // the level expected by that region, but do not update masm.framePushed() + // as that will happen as compilation leaves the block. + // + // Note these operate directly on the stack pointer register. + + void popStackBeforeBranch(StackHeight destStackHeight, + uint32_t stackResultBytes) { + uint32_t framePushedHere = masm.framePushed(); + StackHeight heightThere = + StackHeight(destStackHeight.height + stackResultBytes); + uint32_t framePushedThere = framePushedForHeight(heightThere); + if (framePushedHere > framePushedThere) { + masm.addToStackPtr(Imm32(framePushedHere - framePushedThere)); + } + } + + void popStackBeforeBranch(StackHeight destStackHeight, ResultType type) { + popStackBeforeBranch(destStackHeight, + ABIResultIter::MeasureStackBytes(type)); + } + + // Given that there are |stackParamSize| bytes on the dynamic stack + // corresponding to the stack results, return the stack height once these + // parameters are popped. + + StackHeight stackResultsBase(uint32_t stackParamSize) { + return StackHeight(currentStackHeight() - stackParamSize); + } + + // For most of WebAssembly, adjacent instructions have fallthrough control + // flow between them, which allows us to simply thread the current stack + // height through the compiler. There are two exceptions to this rule: when + // leaving a block via dead code, and when entering the "else" arm of an "if". + // In these cases, the stack height is the block entry height, plus any stack + // values (results in the block exit case, parameters in the else entry case). + + void resetStackHeight(StackHeight destStackHeight, ResultType type) { + uint32_t height = destStackHeight.height; + height += ABIResultIter::MeasureStackBytes(type); + setStackHeight(StackHeight(height)); + } + + // Return offset of stack result. + + uint32_t locateStackResult(const ABIResult& result, StackHeight stackBase, + uint32_t stackResultBytes) { + MOZ_ASSERT(result.onStack()); + MOZ_ASSERT(result.stackOffset() + result.size() <= stackResultBytes); + uint32_t end = computeHeightWithStackResults(stackBase, stackResultBytes); + return end - result.stackOffset(); + } + + public: + ////////////////////////////////////////////////////////////////////// + // + // The Argument area - for outgoing calls. + // + // We abstract these operations as an optimization: we can merge the freeing + // of the argument area and dropping values off the stack after a call. But + // they always amount to manipulating the real stack pointer by some amount. + // + // Note that we do not update currentStackHeight_ for this; the frame does + // not know about outgoing arguments. But we do update framePushed(), so we + // can still index into the frame below the outgoing arguments area. + + // This is always equivalent to a masm.reserveStack() call. + + void allocArgArea(size_t argSize) { + if (argSize) { + masm.reserveStack(argSize); + } + } + + // This frees the argument area allocated by allocArgArea(), and `argSize` + // must be equal to the `argSize` argument to allocArgArea(). In addition + // we drop some values from the frame, corresponding to the values that were + // consumed by the call. + + void freeArgAreaAndPopBytes(size_t argSize, size_t dropSize) { +#ifdef RABALDR_CHUNKY_STACK + // Freeing the outgoing arguments and freeing the consumed values have + // different semantics here, which is why the operation is split. + if (argSize) { + masm.freeStack(argSize); + } + popChunkyBytes(dropSize); +#else + if (argSize + dropSize) { + masm.freeStack(argSize + dropSize); + } +#endif + } +}; + +class BaseStackFrame final : public BaseStackFrameAllocator { + MacroAssembler& masm; + + // The largest observed value of masm.framePushed(), ie, the size of the + // stack frame. Read this for its true value only when code generation is + // finished. + uint32_t maxFramePushed_; + + // Patch point where we check for stack overflow. + CodeOffset stackAddOffset_; + + // Low byte offset of pointer to stack results, if any. + Maybe stackResultsPtrOffset_; + + // The offset of instance pointer. + uint32_t instancePointerOffset_; + + // Low byte offset of local area for true locals (not parameters). + uint32_t varLow_; + + // High byte offset + 1 of local area for true locals. + uint32_t varHigh_; + + // The stack pointer, cached for brevity. + RegisterOrSP sp_; + + public: + explicit BaseStackFrame(MacroAssembler& masm) + : BaseStackFrameAllocator(masm), + masm(masm), + maxFramePushed_(0), + stackAddOffset_(0), + instancePointerOffset_(UINT32_MAX), + varLow_(UINT32_MAX), + varHigh_(UINT32_MAX), + sp_(masm.getStackPointer()) {} + + /////////////////////////////////////////////////////////////////////////// + // + // Stack management and overflow checking + + // This must be called once beginFunction has allocated space for the Header + // (the Frame and DebugFrame) and the Local area, and will record the current + // frame size for internal use by the stack abstractions. + + void onFixedStackAllocated() { + maxFramePushed_ = masm.framePushed(); + BaseStackFrameAllocator::onFixedStackAllocated(); + } + + // We won't know until after we've generated code how big the frame will be + // (we may need arbitrary spill slots and outgoing param slots) so emit a + // patchable add that is patched in endFunction(). + // + // Note the platform scratch register may be used by branchPtr(), so + // generally tmp must be something else. + + void checkStack(Register tmp, BytecodeOffset trapOffset) { + stackAddOffset_ = masm.sub32FromStackPtrWithPatch(tmp); + Label ok; + masm.branchPtr(Assembler::Below, + Address(InstanceReg, wasm::Instance::offsetOfStackLimit()), + tmp, &ok); + masm.wasmTrap(Trap::StackOverflow, trapOffset); + masm.bind(&ok); + } + + void patchCheckStack() { + masm.patchSub32FromStackPtr(stackAddOffset_, + Imm32(int32_t(maxFramePushed_))); + } + + // Very large frames are implausible, probably an attack. + + bool checkStackHeight() { return maxFramePushed_ <= MaxFrameSize; } + + /////////////////////////////////////////////////////////////////////////// + // + // Local area + + struct Local { + // Type of the value. + const MIRType type; + + // Byte offset from Frame "into" the locals, ie positive for true locals + // and negative for incoming args that read directly from the arg area. + // It assumes the stack is growing down and that locals are on the stack + // at lower addresses than Frame, and is the offset from Frame of the + // lowest-addressed byte of the local. + const int32_t offs; + + Local(MIRType type, int32_t offs) : type(type), offs(offs) {} + + bool isStackArgument() const { return offs < 0; } + }; + + // Profiling shows that the number of parameters and locals frequently + // touches or exceeds 8. So 16 seems like a reasonable starting point. + using LocalVector = Vector; + + // Initialize `localInfo` based on the types of `locals` and `args`. + [[nodiscard]] bool setupLocals(const ValTypeVector& locals, + const ArgTypeVector& args, bool debugEnabled, + LocalVector* localInfo) { + if (!localInfo->reserve(locals.length())) { + return false; + } + + DebugOnly index = 0; + BaseLocalIter i(locals, args, debugEnabled); + for (; !i.done() && i.index() < args.lengthWithoutStackResults(); i++) { + MOZ_ASSERT(i.isArg()); + MOZ_ASSERT(i.index() == index); + localInfo->infallibleEmplaceBack(i.mirType(), i.frameOffset()); + index++; + } + + varLow_ = i.frameSize(); + for (; !i.done(); i++) { + MOZ_ASSERT(!i.isArg()); + MOZ_ASSERT(i.index() == index); + localInfo->infallibleEmplaceBack(i.mirType(), i.frameOffset()); + index++; + } + varHigh_ = i.frameSize(); + + // Reserve an additional stack slot for the instance pointer. + const uint32_t pointerAlignedVarHigh = AlignBytes(varHigh_, sizeof(void*)); + const uint32_t localSize = pointerAlignedVarHigh + sizeof(void*); + instancePointerOffset_ = localSize; + + setLocalSize(AlignBytes(localSize, WasmStackAlignment)); + + if (args.hasSyntheticStackResultPointerArg()) { + stackResultsPtrOffset_ = Some(i.stackResultPointerOffset()); + } + + return true; + } + + void zeroLocals(BaseRegAlloc* ra); + + Address addressOfLocal(const Local& local, uint32_t additionalOffset = 0) { + if (local.isStackArgument()) { + return Address(FramePointer, + stackArgumentOffsetFromFp(local) + additionalOffset); + } + return Address(sp_, localOffsetFromSp(local) + additionalOffset); + } + + void loadLocalI32(const Local& src, RegI32 dest) { + masm.load32(addressOfLocal(src), dest); + } + +#ifndef JS_PUNBOX64 + void loadLocalI64Low(const Local& src, RegI32 dest) { + masm.load32(addressOfLocal(src, INT64LOW_OFFSET), dest); + } + + void loadLocalI64High(const Local& src, RegI32 dest) { + masm.load32(addressOfLocal(src, INT64HIGH_OFFSET), dest); + } +#endif + + void loadLocalI64(const Local& src, RegI64 dest) { + masm.load64(addressOfLocal(src), dest); + } + + void loadLocalRef(const Local& src, RegRef dest) { + masm.loadPtr(addressOfLocal(src), dest); + } + + void loadLocalF64(const Local& src, RegF64 dest) { + masm.loadDouble(addressOfLocal(src), dest); + } + + void loadLocalF32(const Local& src, RegF32 dest) { + masm.loadFloat32(addressOfLocal(src), dest); + } + +#ifdef ENABLE_WASM_SIMD + void loadLocalV128(const Local& src, RegV128 dest) { + masm.loadUnalignedSimd128(addressOfLocal(src), dest); + } +#endif + + void storeLocalI32(RegI32 src, const Local& dest) { + masm.store32(src, addressOfLocal(dest)); + } + + void storeLocalI64(RegI64 src, const Local& dest) { + masm.store64(src, addressOfLocal(dest)); + } + + void storeLocalRef(RegRef src, const Local& dest) { + masm.storePtr(src, addressOfLocal(dest)); + } + + void storeLocalF64(RegF64 src, const Local& dest) { + masm.storeDouble(src, addressOfLocal(dest)); + } + + void storeLocalF32(RegF32 src, const Local& dest) { + masm.storeFloat32(src, addressOfLocal(dest)); + } + +#ifdef ENABLE_WASM_SIMD + void storeLocalV128(RegV128 src, const Local& dest) { + masm.storeUnalignedSimd128(src, addressOfLocal(dest)); + } +#endif + + // Offset off of sp_ for `local`. + int32_t localOffsetFromSp(const Local& local) { + MOZ_ASSERT(!local.isStackArgument()); + return localOffset(local.offs); + } + + // Offset off of frame pointer for `stack argument`. + int32_t stackArgumentOffsetFromFp(const Local& local) { + MOZ_ASSERT(local.isStackArgument()); + return -local.offs; + } + + // The incoming stack result area pointer is for stack results of the function + // being compiled. + void loadIncomingStackResultAreaPtr(RegPtr reg) { + const int32_t offset = stackResultsPtrOffset_.value(); + Address src = offset < 0 ? Address(FramePointer, -offset) + : Address(sp_, stackOffset(offset)); + masm.loadPtr(src, reg); + } + + void storeIncomingStackResultAreaPtr(RegPtr reg) { + // If we get here, that means the pointer to the stack results area was + // passed in as a register, and therefore it will be spilled below the + // frame, so the offset is a positive height. + MOZ_ASSERT(stackResultsPtrOffset_.value() > 0); + masm.storePtr(reg, + Address(sp_, stackOffset(stackResultsPtrOffset_.value()))); + } + + void loadInstancePtr(Register dst) { + masm.loadPtr(Address(sp_, stackOffset(instancePointerOffset_)), dst); + } + + void storeInstancePtr(Register instance) { + masm.storePtr(instance, Address(sp_, stackOffset(instancePointerOffset_))); + } + + int32_t getInstancePtrOffset() { return stackOffset(instancePointerOffset_); } + + // An outgoing stack result area pointer is for stack results of callees of + // the function being compiled. + void computeOutgoingStackResultAreaPtr(const StackResultsLoc& results, + RegPtr dest) { + MOZ_ASSERT(results.height() <= masm.framePushed()); + uint32_t offsetFromSP = masm.framePushed() - results.height(); + masm.moveStackPtrTo(dest); + if (offsetFromSP) { + masm.addPtr(Imm32(offsetFromSP), dest); + } + } + + private: + // Offset off of sp_ for a local with offset `offset` from Frame. + int32_t localOffset(int32_t offset) { return masm.framePushed() - offset; } + + public: + /////////////////////////////////////////////////////////////////////////// + // + // Dynamic area + + static constexpr size_t StackSizeOfPtr = ABIResult::StackSizeOfPtr; + static constexpr size_t StackSizeOfInt64 = ABIResult::StackSizeOfInt64; + static constexpr size_t StackSizeOfFloat = ABIResult::StackSizeOfFloat; + static constexpr size_t StackSizeOfDouble = ABIResult::StackSizeOfDouble; +#ifdef ENABLE_WASM_SIMD + static constexpr size_t StackSizeOfV128 = ABIResult::StackSizeOfV128; +#endif + + // Pushes the register `r` to the stack. This pushes the full 64-bit width on + // 64-bit systems, and 32-bits otherwise. + uint32_t pushGPR(Register r) { + DebugOnly stackBefore = currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + pushChunkyBytes(StackSizeOfPtr); + masm.storePtr(r, Address(sp_, stackOffset(currentStackHeight()))); +#else + masm.Push(r); +#endif + maxFramePushed_ = std::max(maxFramePushed_, masm.framePushed()); + MOZ_ASSERT(stackBefore + StackSizeOfPtr == currentStackHeight()); + return currentStackHeight(); + } + + uint32_t pushFloat32(FloatRegister r) { + DebugOnly stackBefore = currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + pushChunkyBytes(StackSizeOfFloat); + masm.storeFloat32(r, Address(sp_, stackOffset(currentStackHeight()))); +#else + masm.Push(r); +#endif + maxFramePushed_ = std::max(maxFramePushed_, masm.framePushed()); + MOZ_ASSERT(stackBefore + StackSizeOfFloat == currentStackHeight()); + return currentStackHeight(); + } + +#ifdef ENABLE_WASM_SIMD + uint32_t pushV128(RegV128 r) { + DebugOnly stackBefore = currentStackHeight(); +# ifdef RABALDR_CHUNKY_STACK + pushChunkyBytes(StackSizeOfV128); +# else + masm.adjustStack(-(int)StackSizeOfV128); +# endif + masm.storeUnalignedSimd128(r, + Address(sp_, stackOffset(currentStackHeight()))); + maxFramePushed_ = std::max(maxFramePushed_, masm.framePushed()); + MOZ_ASSERT(stackBefore + StackSizeOfV128 == currentStackHeight()); + return currentStackHeight(); + } +#endif + + uint32_t pushDouble(FloatRegister r) { + DebugOnly stackBefore = currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + pushChunkyBytes(StackSizeOfDouble); + masm.storeDouble(r, Address(sp_, stackOffset(currentStackHeight()))); +#else + masm.Push(r); +#endif + maxFramePushed_ = std::max(maxFramePushed_, masm.framePushed()); + MOZ_ASSERT(stackBefore + StackSizeOfDouble == currentStackHeight()); + return currentStackHeight(); + } + + // Pops the stack into the register `r`. This pops the full 64-bit width on + // 64-bit systems, and 32-bits otherwise. + void popGPR(Register r) { + DebugOnly stackBefore = currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + masm.loadPtr(Address(sp_, stackOffset(currentStackHeight())), r); + popChunkyBytes(StackSizeOfPtr); +#else + masm.Pop(r); +#endif + MOZ_ASSERT(stackBefore - StackSizeOfPtr == currentStackHeight()); + } + + void popFloat32(FloatRegister r) { + DebugOnly stackBefore = currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + masm.loadFloat32(Address(sp_, stackOffset(currentStackHeight())), r); + popChunkyBytes(StackSizeOfFloat); +#else + masm.Pop(r); +#endif + MOZ_ASSERT(stackBefore - StackSizeOfFloat == currentStackHeight()); + } + + void popDouble(FloatRegister r) { + DebugOnly stackBefore = currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + masm.loadDouble(Address(sp_, stackOffset(currentStackHeight())), r); + popChunkyBytes(StackSizeOfDouble); +#else + masm.Pop(r); +#endif + MOZ_ASSERT(stackBefore - StackSizeOfDouble == currentStackHeight()); + } + +#ifdef ENABLE_WASM_SIMD + void popV128(RegV128 r) { + DebugOnly stackBefore = currentStackHeight(); + masm.loadUnalignedSimd128(Address(sp_, stackOffset(currentStackHeight())), + r); +# ifdef RABALDR_CHUNKY_STACK + popChunkyBytes(StackSizeOfV128); +# else + masm.adjustStack((int)StackSizeOfV128); +# endif + MOZ_ASSERT(stackBefore - StackSizeOfV128 == currentStackHeight()); + } +#endif + + void popBytes(size_t bytes) { + if (bytes > 0) { +#ifdef RABALDR_CHUNKY_STACK + popChunkyBytes(bytes); +#else + masm.freeStack(bytes); +#endif + } + } + + void loadStackI32(int32_t offset, RegI32 dest) { + masm.load32(Address(sp_, stackOffset(offset)), dest); + } + + void loadStackI64(int32_t offset, RegI64 dest) { + masm.load64(Address(sp_, stackOffset(offset)), dest); + } + +#ifndef JS_PUNBOX64 + void loadStackI64Low(int32_t offset, RegI32 dest) { + masm.load32(Address(sp_, stackOffset(offset - INT64LOW_OFFSET)), dest); + } + + void loadStackI64High(int32_t offset, RegI32 dest) { + masm.load32(Address(sp_, stackOffset(offset - INT64HIGH_OFFSET)), dest); + } +#endif + + void loadStackRef(int32_t offset, RegRef dest) { + masm.loadPtr(Address(sp_, stackOffset(offset)), dest); + } + + void loadStackF64(int32_t offset, RegF64 dest) { + masm.loadDouble(Address(sp_, stackOffset(offset)), dest); + } + + void loadStackF32(int32_t offset, RegF32 dest) { + masm.loadFloat32(Address(sp_, stackOffset(offset)), dest); + } + +#ifdef ENABLE_WASM_SIMD + void loadStackV128(int32_t offset, RegV128 dest) { + masm.loadUnalignedSimd128(Address(sp_, stackOffset(offset)), dest); + } +#endif + + uint32_t prepareStackResultArea(StackHeight stackBase, + uint32_t stackResultBytes) { + uint32_t end = computeHeightWithStackResults(stackBase, stackResultBytes); + if (currentStackHeight() < end) { + uint32_t bytes = end - currentStackHeight(); +#ifdef RABALDR_CHUNKY_STACK + pushChunkyBytes(bytes); +#else + masm.reserveStack(bytes); +#endif + maxFramePushed_ = std::max(maxFramePushed_, masm.framePushed()); + } + return end; + } + + void finishStackResultArea(StackHeight stackBase, uint32_t stackResultBytes) { + uint32_t end = computeHeightWithStackResults(stackBase, stackResultBytes); + MOZ_ASSERT(currentStackHeight() >= end); + popBytes(currentStackHeight() - end); + } + + // |srcHeight| and |destHeight| are stack heights *including* |bytes|. + void shuffleStackResultsTowardFP(uint32_t srcHeight, uint32_t destHeight, + uint32_t bytes, Register temp) { + MOZ_ASSERT(destHeight < srcHeight); + MOZ_ASSERT(bytes % sizeof(uint32_t) == 0); + uint32_t destOffset = stackOffset(destHeight) + bytes; + uint32_t srcOffset = stackOffset(srcHeight) + bytes; + while (bytes >= sizeof(intptr_t)) { + destOffset -= sizeof(intptr_t); + srcOffset -= sizeof(intptr_t); + bytes -= sizeof(intptr_t); + masm.loadPtr(Address(sp_, srcOffset), temp); + masm.storePtr(temp, Address(sp_, destOffset)); + } + if (bytes) { + MOZ_ASSERT(bytes == sizeof(uint32_t)); + destOffset -= sizeof(uint32_t); + srcOffset -= sizeof(uint32_t); + masm.load32(Address(sp_, srcOffset), temp); + masm.store32(temp, Address(sp_, destOffset)); + } + } + + // Unlike the overload that operates on raw heights, |srcHeight| and + // |destHeight| are stack heights *not including* |bytes|. + void shuffleStackResultsTowardFP(StackHeight srcHeight, + StackHeight destHeight, uint32_t bytes, + Register temp) { + MOZ_ASSERT(srcHeight.isValid()); + MOZ_ASSERT(destHeight.isValid()); + uint32_t src = computeHeightWithStackResults(srcHeight, bytes); + uint32_t dest = computeHeightWithStackResults(destHeight, bytes); + MOZ_ASSERT(src <= currentStackHeight()); + MOZ_ASSERT(dest <= currentStackHeight()); + shuffleStackResultsTowardFP(src, dest, bytes, temp); + } + + // |srcHeight| and |destHeight| are stack heights *including* |bytes|. + void shuffleStackResultsTowardSP(uint32_t srcHeight, uint32_t destHeight, + uint32_t bytes, Register temp) { + MOZ_ASSERT(destHeight > srcHeight); + MOZ_ASSERT(bytes % sizeof(uint32_t) == 0); + uint32_t destOffset = stackOffset(destHeight); + uint32_t srcOffset = stackOffset(srcHeight); + while (bytes >= sizeof(intptr_t)) { + masm.loadPtr(Address(sp_, srcOffset), temp); + masm.storePtr(temp, Address(sp_, destOffset)); + destOffset += sizeof(intptr_t); + srcOffset += sizeof(intptr_t); + bytes -= sizeof(intptr_t); + } + if (bytes) { + MOZ_ASSERT(bytes == sizeof(uint32_t)); + masm.load32(Address(sp_, srcOffset), temp); + masm.store32(temp, Address(sp_, destOffset)); + } + } + + // Copy results from the top of the current stack frame to an area of memory, + // and pop the stack accordingly. `dest` is the address of the low byte of + // that memory. + void popStackResultsToMemory(Register dest, uint32_t bytes, Register temp) { + MOZ_ASSERT(bytes <= currentStackHeight()); + MOZ_ASSERT(bytes % sizeof(uint32_t) == 0); + uint32_t bytesToPop = bytes; + uint32_t srcOffset = stackOffset(currentStackHeight()); + uint32_t destOffset = 0; + while (bytes >= sizeof(intptr_t)) { + masm.loadPtr(Address(sp_, srcOffset), temp); + masm.storePtr(temp, Address(dest, destOffset)); + destOffset += sizeof(intptr_t); + srcOffset += sizeof(intptr_t); + bytes -= sizeof(intptr_t); + } + if (bytes) { + MOZ_ASSERT(bytes == sizeof(uint32_t)); + masm.load32(Address(sp_, srcOffset), temp); + masm.store32(temp, Address(dest, destOffset)); + } + popBytes(bytesToPop); + } + + void allocArgArea(size_t argSize) { + if (argSize) { + BaseStackFrameAllocator::allocArgArea(argSize); + maxFramePushed_ = std::max(maxFramePushed_, masm.framePushed()); + } + } + + private: + void store32BitsToStack(int32_t imm, uint32_t destHeight, Register temp) { + masm.move32(Imm32(imm), temp); + masm.store32(temp, Address(sp_, stackOffset(destHeight))); + } + + void store64BitsToStack(int64_t imm, uint32_t destHeight, Register temp) { +#ifdef JS_PUNBOX64 + masm.move64(Imm64(imm), Register64(temp)); + masm.store64(Register64(temp), Address(sp_, stackOffset(destHeight))); +#else + union { + int64_t i64; + int32_t i32[2]; + } bits = {.i64 = imm}; + static_assert(sizeof(bits) == 8); + store32BitsToStack(bits.i32[0], destHeight, temp); + store32BitsToStack(bits.i32[1], destHeight - sizeof(int32_t), temp); +#endif + } + + public: + void storeImmediatePtrToStack(intptr_t imm, uint32_t destHeight, + Register temp) { +#ifdef JS_PUNBOX64 + static_assert(StackSizeOfPtr == 8); + store64BitsToStack(imm, destHeight, temp); +#else + static_assert(StackSizeOfPtr == 4); + store32BitsToStack(int32_t(imm), destHeight, temp); +#endif + } + + void storeImmediateI64ToStack(int64_t imm, uint32_t destHeight, + Register temp) { + store64BitsToStack(imm, destHeight, temp); + } + + void storeImmediateF32ToStack(float imm, uint32_t destHeight, Register temp) { + union { + int32_t i32; + float f32; + } bits = {.f32 = imm}; + static_assert(sizeof(bits) == 4); + // Do not store 4 bytes if StackSizeOfFloat == 8. It's probably OK to do + // so, but it costs little to store something predictable. + if (StackSizeOfFloat == 4) { + store32BitsToStack(bits.i32, destHeight, temp); + } else { + store64BitsToStack(uint32_t(bits.i32), destHeight, temp); + } + } + + void storeImmediateF64ToStack(double imm, uint32_t destHeight, + Register temp) { + union { + int64_t i64; + double f64; + } bits = {.f64 = imm}; + static_assert(sizeof(bits) == 8); + store64BitsToStack(bits.i64, destHeight, temp); + } + +#ifdef ENABLE_WASM_SIMD + void storeImmediateV128ToStack(V128 imm, uint32_t destHeight, Register temp) { + union { + int32_t i32[4]; + uint8_t bytes[16]; + } bits{}; + static_assert(sizeof(bits) == 16); + memcpy(bits.bytes, imm.bytes, 16); + for (unsigned i = 0; i < 4; i++) { + store32BitsToStack(bits.i32[i], destHeight - i * sizeof(int32_t), temp); + } + } +#endif +}; + +////////////////////////////////////////////////////////////////////////////// +// +// MachineStackTracker, used for stack-slot pointerness tracking. + +// An expensive operation in stack-map creation is copying of the +// MachineStackTracker (MST) into the final StackMap. This is done in +// StackMapGenerator::createStackMap. Given that this is basically a +// bit-array copy, it is reasonable to ask whether the two classes could have +// a more similar representation, so that the copy could then be done with +// `memcpy`. +// +// Although in principle feasible, the follow complications exist, and so for +// the moment, this has not been done. +// +// * StackMap is optimised for compact size (storage) since there will be +// many, so it uses a true bitmap. MST is intended to be fast and simple, +// and only one exists at once (per compilation thread). Doing this would +// require MST to use a true bitmap, and hence .. +// +// * .. the copying can't be a straight memcpy, since StackMap has entries for +// words not covered by MST. Hence the copy would need to shift bits in +// each byte left or right (statistically speaking, in 7 cases out of 8) in +// order to ensure no "holes" in the resulting bitmap. +// +// * Furthermore the copying would need to logically invert the direction of +// the stacks. For MST, index zero in the vector corresponds to the highest +// address in the stack. For StackMap, bit index zero corresponds to the +// lowest address in the stack. +// +// * Finally, StackMap is a variable-length structure whose size must be known +// at creation time. The size of an MST by contrast isn't known at creation +// time -- it grows as the baseline compiler pushes stuff on its value +// stack. That's why it has to have vector entry 0 being the highest address. +// +// * Although not directly relevant, StackMaps are also created by the via-Ion +// compilation routes, by translation from the pre-existing "JS-era" +// LSafePoints (CreateStackMapFromLSafepoint). So if we want to mash +// StackMap around to suit baseline better, we also need to ensure it +// doesn't break Ion somehow. + +class MachineStackTracker { + // Simulates the machine's stack, with one bool per word. The booleans are + // represented as `uint8_t`s so as to guarantee the element size is one + // byte. Index zero in this vector corresponds to the highest address in + // the machine's stack. The last entry corresponds to what SP currently + // points at. This all assumes a grow-down stack. + // + // numPtrs_ contains the number of "true" values in vec_, and is therefore + // redundant. But it serves as a constant-time way to detect the common + // case where vec_ holds no "true" values. + size_t numPtrs_; + Vector vec_; + + public: + MachineStackTracker() : numPtrs_(0) {} + + ~MachineStackTracker() { +#ifdef DEBUG + size_t n = 0; + for (uint8_t b : vec_) { + n += (b ? 1 : 0); + } + MOZ_ASSERT(n == numPtrs_); +#endif + } + + // Clone this MachineStackTracker, writing the result at |dst|. + [[nodiscard]] bool cloneTo(MachineStackTracker* dst); + + // Notionally push |n| non-pointers on the stack. + [[nodiscard]] bool pushNonGCPointers(size_t n) { + return vec_.appendN(uint8_t(false), n); + } + + // Mark the stack slot |offsetFromSP| up from the bottom as holding a + // pointer. + void setGCPointer(size_t offsetFromSP) { + // offsetFromSP == 0 denotes the most recently pushed item, == 1 the + // second most recently pushed item, etc. + MOZ_ASSERT(offsetFromSP < vec_.length()); + + size_t offsetFromTop = vec_.length() - 1 - offsetFromSP; + numPtrs_ = numPtrs_ + 1 - (vec_[offsetFromTop] ? 1 : 0); + vec_[offsetFromTop] = uint8_t(true); + } + + // Query the pointerness of the slot |offsetFromSP| up from the bottom. + bool isGCPointer(size_t offsetFromSP) const { + MOZ_ASSERT(offsetFromSP < vec_.length()); + + size_t offsetFromTop = vec_.length() - 1 - offsetFromSP; + return bool(vec_[offsetFromTop]); + } + + // Return the number of words tracked by this MachineStackTracker. + size_t length() const { return vec_.length(); } + + // Return the number of pointer-typed words tracked by this + // MachineStackTracker. + size_t numPtrs() const { + MOZ_ASSERT(numPtrs_ <= length()); + return numPtrs_; + } + + // Discard all contents, but (per mozilla::Vector::clear semantics) don't + // free or reallocate any dynamic storage associated with |vec_|. + void clear() { + vec_.clear(); + numPtrs_ = 0; + } + + // An iterator that produces indices of reftyped slots, starting at the + // logical bottom of the (grow-down) stack. Indices have the same meaning + // as the arguments to `isGCPointer`. That is, if this iterator produces a + // value `i`, then it means that `isGCPointer(i) == true`; if the value `i` + // is never produced then `isGCPointer(i) == false`. The values are + // produced in ascending order. + // + // Because most slots are non-reftyped, some effort has been put into + // skipping over large groups of non-reftyped slots quickly. + class Iter { + // Both `bufU8_` and `bufU32_` are made to point to `vec_`s array of + // `uint8_t`s, so we can scan (backwards) through it either in bytes or + // 32-bit words. Recall that the last element in `vec_` pertains to the + // lowest-addressed word in the machine's grow-down stack, and we want to + // iterate logically "up" this stack, so we need to iterate backwards + // through `vec_`. + // + // This dual-pointer scheme assumes that the `vec_`s content array is at + // least 32-bit aligned. + const uint8_t* bufU8_; + const uint32_t* bufU32_; + // The number of elements in `bufU8_`. + const size_t nElems_; + // The index in `bufU8_` where the next search should start. + size_t next_; + + public: + explicit Iter(const MachineStackTracker& mst) + : bufU8_((uint8_t*)mst.vec_.begin()), + bufU32_((uint32_t*)mst.vec_.begin()), + nElems_(mst.vec_.length()), + next_(mst.vec_.length() - 1) { + MOZ_ASSERT(uintptr_t(bufU8_) == uintptr_t(bufU32_)); + // Check minimum alignment constraint on the array. + MOZ_ASSERT(0 == (uintptr_t(bufU8_) & 3)); + } + + ~Iter() { MOZ_ASSERT(uintptr_t(bufU8_) == uintptr_t(bufU32_)); } + + // It is important, for termination of the search loop in `next()`, that + // this has the value obtained by subtracting 1 from size_t(0). + static constexpr size_t FINISHED = ~size_t(0); + static_assert(FINISHED == size_t(0) - 1); + + // Returns the next index `i` for which `isGCPointer(i) == true`. + size_t get() { + while (next_ != FINISHED) { + if (bufU8_[next_]) { + next_--; + return nElems_ - 1 - (next_ + 1); + } + // Invariant: next_ != FINISHED (so it's still a valid index) + // and: bufU8_[next_] == 0 + // (so we need to move backwards by at least 1) + // + // BEGIN optimization -- this could be removed without affecting + // correctness. + if ((next_ & 7) == 0) { + // We're at the "bottom" of the current dual-4-element word. Check + // if we can jump backwards by 8. This saves a conditional branch + // and a few cycles by ORing two adjacent 32-bit words together, + // whilst not requiring 64-bit alignment of `bufU32_`. + while (next_ >= 8 && + (bufU32_[(next_ - 4) >> 2] | bufU32_[(next_ - 8) >> 2]) == 0) { + next_ -= 8; + } + } + // END optimization + next_--; + } + return FINISHED; + } + }; +}; + +////////////////////////////////////////////////////////////////////////////// +// +// StackMapGenerator, which carries all state needed to create stackmaps. + +enum class HasDebugFrameWithLiveRefs { No, Maybe }; + +struct StackMapGenerator { + private: + // --- These are constant for the life of the function's compilation --- + + // For generating stackmaps, we'll need to know the offsets of registers + // as saved by the trap exit stub. + const RegisterOffsets& trapExitLayout_; + const size_t trapExitLayoutNumWords_; + + // Completed stackmaps are added here + StackMaps* stackMaps_; + + // So as to be able to get current offset when creating stackmaps + const MacroAssembler& masm_; + + public: + // --- These are constant once we've completed beginFunction() --- + + // The number of words of arguments passed to this function in memory. + size_t numStackArgWords; + + MachineStackTracker machineStackTracker; // tracks machine stack pointerness + + // This holds masm.framePushed at entry to the function's body. It is a + // Maybe because createStackMap needs to know whether or not we're still + // in the prologue. It makes a Nothing-to-Some transition just once per + // function. + Maybe framePushedAtEntryToBody; + + // --- These can change at any point --- + + // This holds masm.framePushed at it would be be for a function call + // instruction, but excluding the stack area used to pass arguments in + // memory. That is, for an upcoming function call, this will hold + // + // masm.framePushed() at the call instruction - + // StackArgAreaSizeUnaligned(argumentTypes) + // + // This value denotes the lowest-addressed stack word covered by the current + // function's stackmap. Words below this point form the highest-addressed + // area of the callee's stackmap. Note that all alignment padding above the + // arguments-in-memory themselves belongs to the caller's stackmap, which + // is why this is defined in terms of StackArgAreaSizeUnaligned() rather than + // StackArgAreaSizeAligned(). + // + // When not inside a function call setup/teardown sequence, it is Nothing. + // It can make Nothing-to/from-Some transitions arbitrarily as we progress + // through the function body. + Maybe framePushedExcludingOutboundCallArgs; + + // The number of memory-resident, ref-typed entries on the containing + // BaseCompiler::stk_. + size_t memRefsOnStk; + + // This is a copy of machineStackTracker that is used only within individual + // calls to createStackMap. It is here only to avoid possible heap allocation + // costs resulting from making it local to createStackMap(). + MachineStackTracker augmentedMst; + + StackMapGenerator(StackMaps* stackMaps, const RegisterOffsets& trapExitLayout, + const size_t trapExitLayoutNumWords, + const MacroAssembler& masm) + : trapExitLayout_(trapExitLayout), + trapExitLayoutNumWords_(trapExitLayoutNumWords), + stackMaps_(stackMaps), + masm_(masm), + numStackArgWords(0), + memRefsOnStk(0) {} + + // At the beginning of a function, we may have live roots in registers (as + // arguments) at the point where we perform a stack overflow check. This + // method generates the "extra" stackmap entries to describe that, in the + // case that the check fails and we wind up calling into the wasm exit + // stub, as generated by GenerateTrapExit(). + // + // The resulting map must correspond precisely with the stack layout + // created for the integer registers as saved by (code generated by) + // GenerateTrapExit(). To do that we use trapExitLayout_ and + // trapExitLayoutNumWords_, which together comprise a description of the + // layout and are created by GenerateTrapExitRegisterOffsets(). + [[nodiscard]] bool generateStackmapEntriesForTrapExit( + const ArgTypeVector& args, ExitStubMapVector* extras); + + // Creates a stackmap associated with the instruction denoted by + // |assemblerOffset|, incorporating pointers from the current operand + // stack |stk|, incorporating possible extra pointers in |extra| at the + // lower addressed end, and possibly with the associated frame having a + // DebugFrame that must be traced, as indicated by |debugFrameWithLiveRefs|. + [[nodiscard]] bool createStackMap( + const char* who, const ExitStubMapVector& extras, + uint32_t assemblerOffset, + HasDebugFrameWithLiveRefs debugFrameWithLiveRefs, const StkVector& stk); +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_frame_h diff --git a/js/src/wasm/WasmBCMemory.cpp b/js/src/wasm/WasmBCMemory.cpp new file mode 100644 index 0000000000..a8777a4217 --- /dev/null +++ b/js/src/wasm/WasmBCMemory.cpp @@ -0,0 +1,2799 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmBCClass.h" +#include "wasm/WasmBCDefs.h" +#include "wasm/WasmBCRegDefs.h" + +#include "jit/MacroAssembler-inl.h" + +#include "wasm/WasmBCClass-inl.h" +#include "wasm/WasmBCCodegen-inl.h" +#include "wasm/WasmBCRegDefs-inl.h" +#include "wasm/WasmBCRegMgmt-inl.h" +#include "wasm/WasmBCStkMgmt-inl.h" + +namespace js { +namespace wasm { + +////////////////////////////////////////////////////////////////////////////// +// +// Heap access subroutines. + +// Bounds check elimination. +// +// We perform BCE on two kinds of address expressions: on constant heap pointers +// that are known to be in the heap or will be handled by the out-of-bounds trap +// handler; and on local variables that have been checked in dominating code +// without being updated since. +// +// For an access through a constant heap pointer + an offset we can eliminate +// the bounds check if the sum of the address and offset is below the sum of the +// minimum memory length and the offset guard length. +// +// For an access through a local variable + an offset we can eliminate the +// bounds check if the local variable has already been checked and has not been +// updated since, and the offset is less than the guard limit. +// +// To track locals for which we can eliminate checks we use a bit vector +// bceSafe_ that has a bit set for those locals whose bounds have been checked +// and which have not subsequently been set. Initially this vector is zero. +// +// In straight-line code a bit is set when we perform a bounds check on an +// access via the local and is reset when the variable is updated. +// +// In control flow, the bit vector is manipulated as follows. Each ControlItem +// has a value bceSafeOnEntry, which is the value of bceSafe_ on entry to the +// item, and a value bceSafeOnExit, which is initially ~0. On a branch (br, +// brIf, brTable), we always AND the branch target's bceSafeOnExit with the +// value of bceSafe_ at the branch point. On exiting an item by falling out of +// it, provided we're not in dead code, we AND the current value of bceSafe_ +// into the item's bceSafeOnExit. Additional processing depends on the item +// type: +// +// - After a block, set bceSafe_ to the block's bceSafeOnExit. +// +// - On loop entry, after pushing the ControlItem, set bceSafe_ to zero; the +// back edges would otherwise require us to iterate to a fixedpoint. +// +// - After a loop, the bceSafe_ is left unchanged, because only fallthrough +// control flow will reach that point and the bceSafe_ value represents the +// correct state of the fallthrough path. +// +// - Set bceSafe_ to the ControlItem's bceSafeOnEntry at both the 'then' branch +// and the 'else' branch. +// +// - After an if-then-else, set bceSafe_ to the if-then-else's bceSafeOnExit. +// +// - After an if-then, set bceSafe_ to the if-then's bceSafeOnExit AND'ed with +// the if-then's bceSafeOnEntry. +// +// Finally, when the debugger allows locals to be mutated we must disable BCE +// for references via a local, by returning immediately from bceCheckLocal if +// compilerEnv_.debugEnabled() is true. + +void BaseCompiler::bceCheckLocal(MemoryAccessDesc* access, AccessCheck* check, + uint32_t local) { + if (local >= sizeof(BCESet) * 8) { + return; + } + + uint32_t offsetGuardLimit = + GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled()); + + if ((bceSafe_ & (BCESet(1) << local)) && + access->offset64() < offsetGuardLimit) { + check->omitBoundsCheck = true; + } + + // The local becomes safe even if the offset is beyond the guard limit. + bceSafe_ |= (BCESet(1) << local); +} + +void BaseCompiler::bceLocalIsUpdated(uint32_t local) { + if (local >= sizeof(BCESet) * 8) { + return; + } + + bceSafe_ &= ~(BCESet(1) << local); +} + +// Alignment check elimination. +// +// Alignment checks for atomic operations can be omitted if the pointer is a +// constant and the pointer + offset is aligned. Alignment checking that can't +// be omitted can still be simplified by checking only the pointer if the offset +// is aligned. +// +// (In addition, alignment checking of the pointer can be omitted if the pointer +// has been checked in dominating code, but we don't do that yet.) + +template <> +RegI32 BaseCompiler::popConstMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check) { + int32_t addrTemp; + MOZ_ALWAYS_TRUE(popConst(&addrTemp)); + uint32_t addr = addrTemp; + + uint32_t offsetGuardLimit = + GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled()); + + uint64_t ea = uint64_t(addr) + uint64_t(access->offset()); + uint64_t limit = moduleEnv_.memory->initialLength32() + offsetGuardLimit; + + check->omitBoundsCheck = ea < limit; + check->omitAlignmentCheck = (ea & (access->byteSize() - 1)) == 0; + + // Fold the offset into the pointer if we can, as this is always + // beneficial. + if (ea <= UINT32_MAX) { + addr = uint32_t(ea); + access->clearOffset(); + } + + RegI32 r = needI32(); + moveImm32(int32_t(addr), r); + return r; +} + +#ifdef ENABLE_WASM_MEMORY64 +template <> +RegI64 BaseCompiler::popConstMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check) { + int64_t addrTemp; + MOZ_ALWAYS_TRUE(popConst(&addrTemp)); + uint64_t addr = addrTemp; + + uint32_t offsetGuardLimit = + GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled()); + + uint64_t ea = addr + access->offset64(); + bool overflow = ea < addr; + uint64_t limit = moduleEnv_.memory->initialLength64() + offsetGuardLimit; + + if (!overflow) { + check->omitBoundsCheck = ea < limit; + check->omitAlignmentCheck = (ea & (access->byteSize() - 1)) == 0; + + // Fold the offset into the pointer if we can, as this is always + // beneficial. + addr = uint64_t(ea); + access->clearOffset(); + } + + RegI64 r = needI64(); + moveImm64(int64_t(addr), r); + return r; +} +#endif + +template +RegType BaseCompiler::popMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check) { + check->onlyPointerAlignment = + (access->offset64() & (access->byteSize() - 1)) == 0; + + // If there's a constant it will have the correct type for RegType. + if (hasConst()) { + return popConstMemoryAccess(access, check); + } + + // If there's a local it will have the correct type for RegType. + uint32_t local; + if (peekLocal(&local)) { + bceCheckLocal(access, check, local); + } + + return pop(); +} + +#ifdef JS_64BIT +static inline RegI64 RegPtrToRegIntptr(RegPtr r) { + return RegI64(Register64(Register(r))); +} + +# ifndef WASM_HAS_HEAPREG +static inline RegPtr RegIntptrToRegPtr(RegI64 r) { + return RegPtr(Register64(r).reg); +} +# endif +#else +static inline RegI32 RegPtrToRegIntptr(RegPtr r) { return RegI32(Register(r)); } + +# ifndef WASM_HAS_HEAPREG +static inline RegPtr RegIntptrToRegPtr(RegI32 r) { return RegPtr(Register(r)); } +# endif +#endif + +#ifdef WASM_HAS_HEAPREG +void BaseCompiler::pushHeapBase() { + RegPtr heapBase = need(); + move(RegPtr(HeapReg), heapBase); + push(RegPtrToRegIntptr(heapBase)); +} +#else +void BaseCompiler::pushHeapBase() { + RegPtr heapBase = need(); +# ifdef RABALDR_PIN_INSTANCE + movePtr(RegPtr(InstanceReg), heapBase); +# else + fr.loadInstancePtr(heapBase); +# endif + masm.loadPtr(Address(heapBase, Instance::offsetOfMemoryBase()), heapBase); + push(RegPtrToRegIntptr(heapBase)); +} +#endif + +void BaseCompiler::branchAddNoOverflow(uint64_t offset, RegI32 ptr, Label* ok) { + // The invariant holds because ptr is RegI32 - this is m32. + MOZ_ASSERT(offset <= UINT32_MAX); + masm.branchAdd32(Assembler::CarryClear, Imm32(uint32_t(offset)), ptr, ok); +} + +#ifdef ENABLE_WASM_MEMORY64 +void BaseCompiler::branchAddNoOverflow(uint64_t offset, RegI64 ptr, Label* ok) { +# if defined(JS_64BIT) + masm.branchAddPtr(Assembler::CarryClear, ImmWord(offset), Register64(ptr).reg, + ok); +# else + masm.branchAdd64(Assembler::CarryClear, Imm64(offset), ptr, ok); +# endif +} +#endif + +void BaseCompiler::branchTestLowZero(RegI32 ptr, Imm32 mask, Label* ok) { + masm.branchTest32(Assembler::Zero, ptr, mask, ok); +} + +#ifdef ENABLE_WASM_MEMORY64 +void BaseCompiler::branchTestLowZero(RegI64 ptr, Imm32 mask, Label* ok) { +# ifdef JS_64BIT + masm.branchTestPtr(Assembler::Zero, Register64(ptr).reg, mask, ok); +# else + masm.branchTestPtr(Assembler::Zero, ptr.low, mask, ok); +# endif +} +#endif + +void BaseCompiler::boundsCheck4GBOrLargerAccess(RegPtr instance, RegI32 ptr, + Label* ok) { +#ifdef JS_64BIT + // Extend the value to 64 bits, check the 64-bit value against the 64-bit + // bound, then chop back to 32 bits. On most platform the extending and + // chopping are no-ops. It's important that the value we end up with has + // flowed through the Spectre mask + + // Note, ptr and ptr64 are the same register. + RegI64 ptr64 = fromI32(ptr); + + // In principle there may be non-zero bits in the upper bits of the + // register; clear them. +# ifdef RABALDR_ZERO_EXTENDS + masm.debugAssertCanonicalInt32(ptr); +# else + masm.move32To64ZeroExtend(ptr, ptr64); +# endif + + boundsCheck4GBOrLargerAccess(instance, ptr64, ok); + + // Restore the value to the canonical form for a 32-bit value in a + // 64-bit register and/or the appropriate form for further use in the + // indexing instruction. +# ifdef RABALDR_ZERO_EXTENDS + // The canonical value is zero-extended; we already have that. +# else + masm.move64To32(ptr64, ptr); +# endif +#else + // No support needed, we have max 2GB heap on 32-bit + MOZ_CRASH("No 32-bit support"); +#endif +} + +void BaseCompiler::boundsCheckBelow4GBAccess(RegPtr instance, RegI32 ptr, + Label* ok) { + // If the memory's max size is known to be smaller than 64K pages exactly, + // we can use a 32-bit check and avoid extension and wrapping. + masm.wasmBoundsCheck32( + Assembler::Below, ptr, + Address(instance, Instance::offsetOfBoundsCheckLimit()), ok); +} + +void BaseCompiler::boundsCheck4GBOrLargerAccess(RegPtr instance, RegI64 ptr, + Label* ok) { + // Any Spectre mitigation will appear to update the ptr64 register. + masm.wasmBoundsCheck64( + Assembler::Below, ptr, + Address(instance, Instance::offsetOfBoundsCheckLimit()), ok); +} + +void BaseCompiler::boundsCheckBelow4GBAccess(RegPtr instance, RegI64 ptr, + Label* ok) { + // The bounds check limit is valid to 64 bits, so there's no sense in doing + // anything complicated here. There may be optimization paths here in the + // future and they may differ on 32-bit and 64-bit. + boundsCheck4GBOrLargerAccess(instance, ptr, ok); +} + +// Make sure the ptr could be used as an index register. +static inline void ToValidIndex(MacroAssembler& masm, RegI32 ptr) { +#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) + // When ptr is used as an index, it will be added to a 64-bit register. + // So we should explicitly promote ptr to 64-bit. Since now ptr holds a + // unsigned 32-bit value, we zero-extend it to 64-bit here. + masm.move32To64ZeroExtend(ptr, Register64(ptr)); +#endif +} + +#if defined(ENABLE_WASM_MEMORY64) +static inline void ToValidIndex(MacroAssembler& masm, RegI64 ptr) {} +#endif + +// RegIndexType is RegI32 for Memory32 and RegI64 for Memory64. +template +void BaseCompiler::prepareMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, RegPtr instance, + RegIndexType ptr) { + uint32_t offsetGuardLimit = + GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled()); + + // Fold offset if necessary for further computations. + if (access->offset64() >= offsetGuardLimit || + access->offset64() > UINT32_MAX || + (access->isAtomic() && !check->omitAlignmentCheck && + !check->onlyPointerAlignment)) { + Label ok; + branchAddNoOverflow(access->offset64(), ptr, &ok); + masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&ok); + access->clearOffset(); + check->onlyPointerAlignment = true; + } + + // Alignment check if required. + + if (access->isAtomic() && !check->omitAlignmentCheck) { + MOZ_ASSERT(check->onlyPointerAlignment); + // We only care about the low pointer bits here. + Label ok; + branchTestLowZero(ptr, Imm32(access->byteSize() - 1), &ok); + masm.wasmTrap(Trap::UnalignedAccess, bytecodeOffset()); + masm.bind(&ok); + } + + // Ensure no instance if we don't need it. + + if (moduleEnv_.hugeMemoryEnabled()) { + // We have HeapReg and no bounds checking and need load neither + // memoryBase nor boundsCheckLimit from instance. + MOZ_ASSERT_IF(check->omitBoundsCheck, instance.isInvalid()); + } +#ifdef WASM_HAS_HEAPREG + // We have HeapReg and don't need to load the memoryBase from instance. + MOZ_ASSERT_IF(check->omitBoundsCheck, instance.isInvalid()); +#endif + + // Bounds check if required. + + if (!moduleEnv_.hugeMemoryEnabled() && !check->omitBoundsCheck) { + Label ok; +#ifdef JS_64BIT + // The checking depends on how many bits are in the pointer and how many + // bits are in the bound. + static_assert(0x100000000 % PageSize == 0); + if (!moduleEnv_.memory->boundsCheckLimitIs32Bits() && + MaxMemoryPages(moduleEnv_.memory->indexType()) >= + Pages(0x100000000 / PageSize)) { + boundsCheck4GBOrLargerAccess(instance, ptr, &ok); + } else { + boundsCheckBelow4GBAccess(instance, ptr, &ok); + } +#else + boundsCheckBelow4GBAccess(instance, ptr, &ok); +#endif + masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&ok); + } + + ToValidIndex(masm, ptr); +} + +template +void BaseCompiler::computeEffectiveAddress(MemoryAccessDesc* access) { + if (access->offset()) { + Label ok; + RegIndexType ptr = pop(); + branchAddNoOverflow(access->offset64(), ptr, &ok); + masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&ok); + access->clearOffset(); + push(ptr); + } +} + +bool BaseCompiler::needInstanceForAccess(const AccessCheck& check) { +#ifndef WASM_HAS_HEAPREG + // Platform requires instance for memory base. + return true; +#else + return !moduleEnv_.hugeMemoryEnabled() && !check.omitBoundsCheck; +#endif +} + +RegPtr BaseCompiler::maybeLoadInstanceForAccess(const AccessCheck& check) { + if (needInstanceForAccess(check)) { +#ifdef RABALDR_PIN_INSTANCE + // NOTE, returning InstanceReg here depends for correctness on *ALL* + // clients not attempting to free this register and not push it on the value + // stack. + // + // We have assertions in place to guard against that, so the risk of the + // leaky abstraction is acceptable. performRegisterLeakCheck() will ensure + // that after every bytecode, the union of available registers from the + // regalloc and used registers from the stack equals the set of allocatable + // registers at startup. Thus if the instance is freed incorrectly it will + // end up in that union via the regalloc, and if it is pushed incorrectly it + // will end up in the union via the stack. + return RegPtr(InstanceReg); +#else + RegPtr instance = need(); + fr.loadInstancePtr(instance); + return instance; +#endif + } + return RegPtr::Invalid(); +} + +RegPtr BaseCompiler::maybeLoadInstanceForAccess(const AccessCheck& check, + RegPtr specific) { + if (needInstanceForAccess(check)) { +#ifdef RABALDR_PIN_INSTANCE + movePtr(RegPtr(InstanceReg), specific); +#else + fr.loadInstancePtr(specific); +#endif + return specific; + } + return RegPtr::Invalid(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Load and store. + +void BaseCompiler::executeLoad(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI32 ptr, AnyReg dest, + RegI32 temp) { + // Emit the load. At this point, 64-bit offsets will have been resolved. +#if defined(JS_CODEGEN_X64) + MOZ_ASSERT(temp.isInvalid()); + Operand srcAddr(HeapReg, ptr, TimesOne, access->offset()); + + if (dest.tag == AnyReg::I64) { + masm.wasmLoadI64(*access, srcAddr, dest.i64()); + } else { + masm.wasmLoad(*access, srcAddr, dest.any()); + } +#elif defined(JS_CODEGEN_X86) + MOZ_ASSERT(temp.isInvalid()); + masm.addPtr(Address(instance, Instance::offsetOfMemoryBase()), ptr); + Operand srcAddr(ptr, access->offset()); + + if (dest.tag == AnyReg::I64) { + MOZ_ASSERT(dest.i64() == specific_.abiReturnRegI64); + masm.wasmLoadI64(*access, srcAddr, dest.i64()); + } else { + // For 8 bit loads, this will generate movsbl or movzbl, so + // there's no constraint on what the output register may be. + masm.wasmLoad(*access, srcAddr, dest.any()); + } +#elif defined(JS_CODEGEN_MIPS64) + if (IsUnaligned(*access)) { + switch (dest.tag) { + case AnyReg::I64: + masm.wasmUnalignedLoadI64(*access, HeapReg, ptr, ptr, dest.i64(), temp); + break; + case AnyReg::F32: + masm.wasmUnalignedLoadFP(*access, HeapReg, ptr, ptr, dest.f32(), temp); + break; + case AnyReg::F64: + masm.wasmUnalignedLoadFP(*access, HeapReg, ptr, ptr, dest.f64(), temp); + break; + case AnyReg::I32: + masm.wasmUnalignedLoad(*access, HeapReg, ptr, ptr, dest.i32(), temp); + break; + default: + MOZ_CRASH("Unexpected type"); + } + } else { + if (dest.tag == AnyReg::I64) { + masm.wasmLoadI64(*access, HeapReg, ptr, ptr, dest.i64()); + } else { + masm.wasmLoad(*access, HeapReg, ptr, ptr, dest.any()); + } + } +#elif defined(JS_CODEGEN_ARM) + MOZ_ASSERT(temp.isInvalid()); + if (dest.tag == AnyReg::I64) { + masm.wasmLoadI64(*access, HeapReg, ptr, ptr, dest.i64()); + } else { + masm.wasmLoad(*access, HeapReg, ptr, ptr, dest.any()); + } +#elif defined(JS_CODEGEN_ARM64) + MOZ_ASSERT(temp.isInvalid()); + if (dest.tag == AnyReg::I64) { + masm.wasmLoadI64(*access, HeapReg, ptr, dest.i64()); + } else { + masm.wasmLoad(*access, HeapReg, ptr, dest.any()); + } +#elif defined(JS_CODEGEN_LOONG64) + MOZ_ASSERT(temp.isInvalid()); + if (dest.tag == AnyReg::I64) { + masm.wasmLoadI64(*access, HeapReg, ptr, ptr, dest.i64()); + } else { + masm.wasmLoad(*access, HeapReg, ptr, ptr, dest.any()); + } +#elif defined(JS_CODEGEN_RISCV64) + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +#else + MOZ_CRASH("BaseCompiler platform hook: load"); +#endif +} + +// ptr and dest may be the same iff dest is I32. +// This may destroy ptr even if ptr and dest are not the same. +void BaseCompiler::load(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI32 ptr, AnyReg dest, RegI32 temp) { + prepareMemoryAccess(access, check, instance, ptr); + executeLoad(access, check, instance, ptr, dest, temp); +} + +#ifdef ENABLE_WASM_MEMORY64 +void BaseCompiler::load(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI64 ptr, AnyReg dest, RegI64 temp) { + prepareMemoryAccess(access, check, instance, ptr); + +# if !defined(JS_64BIT) + // On 32-bit systems we have a maximum 2GB heap and bounds checking has + // been applied to ensure that the 64-bit pointer is valid. + return executeLoad(access, check, instance, RegI32(ptr.low), dest, + maybeFromI64(temp)); +# elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64) + // On x64 and arm64 the 32-bit code simply assumes that the high bits of the + // 64-bit pointer register are zero and performs a 64-bit add. Thus the code + // generated is the same for the 64-bit and the 32-bit case. + return executeLoad(access, check, instance, RegI32(ptr.reg), dest, + maybeFromI64(temp)); +# elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + // On mips64 and loongarch64, the 'prepareMemoryAccess' function will make + // sure that ptr holds a valid 64-bit index value. Thus the code generated in + // 'executeLoad' is the same for the 64-bit and the 32-bit case. + return executeLoad(access, check, instance, RegI32(ptr.reg), dest, + maybeFromI64(temp)); +# else + MOZ_CRASH("Missing platform hook"); +# endif +} +#endif + +void BaseCompiler::executeStore(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI32 ptr, AnyReg src, + RegI32 temp) { + // Emit the store. At this point, 64-bit offsets will have been resolved. +#if defined(JS_CODEGEN_X64) + MOZ_ASSERT(temp.isInvalid()); + Operand dstAddr(HeapReg, ptr, TimesOne, access->offset()); + + masm.wasmStore(*access, src.any(), dstAddr); +#elif defined(JS_CODEGEN_X86) + MOZ_ASSERT(temp.isInvalid()); + masm.addPtr(Address(instance, Instance::offsetOfMemoryBase()), ptr); + Operand dstAddr(ptr, access->offset()); + + if (access->type() == Scalar::Int64) { + masm.wasmStoreI64(*access, src.i64(), dstAddr); + } else { + AnyRegister value; + ScratchI8 scratch(*this); + if (src.tag == AnyReg::I64) { + if (access->byteSize() == 1 && !ra.isSingleByteI32(src.i64().low)) { + masm.mov(src.i64().low, scratch); + value = AnyRegister(scratch); + } else { + value = AnyRegister(src.i64().low); + } + } else if (access->byteSize() == 1 && !ra.isSingleByteI32(src.i32())) { + masm.mov(src.i32(), scratch); + value = AnyRegister(scratch); + } else { + value = src.any(); + } + + masm.wasmStore(*access, value, dstAddr); + } +#elif defined(JS_CODEGEN_ARM) + MOZ_ASSERT(temp.isInvalid()); + if (access->type() == Scalar::Int64) { + masm.wasmStoreI64(*access, src.i64(), HeapReg, ptr, ptr); + } else if (src.tag == AnyReg::I64) { + masm.wasmStore(*access, AnyRegister(src.i64().low), HeapReg, ptr, ptr); + } else { + masm.wasmStore(*access, src.any(), HeapReg, ptr, ptr); + } +#elif defined(JS_CODEGEN_MIPS64) + if (IsUnaligned(*access)) { + switch (src.tag) { + case AnyReg::I64: + masm.wasmUnalignedStoreI64(*access, src.i64(), HeapReg, ptr, ptr, temp); + break; + case AnyReg::F32: + masm.wasmUnalignedStoreFP(*access, src.f32(), HeapReg, ptr, ptr, temp); + break; + case AnyReg::F64: + masm.wasmUnalignedStoreFP(*access, src.f64(), HeapReg, ptr, ptr, temp); + break; + case AnyReg::I32: + masm.wasmUnalignedStore(*access, src.i32(), HeapReg, ptr, ptr, temp); + break; + default: + MOZ_CRASH("Unexpected type"); + } + } else { + if (src.tag == AnyReg::I64) { + masm.wasmStoreI64(*access, src.i64(), HeapReg, ptr, ptr); + } else { + masm.wasmStore(*access, src.any(), HeapReg, ptr, ptr); + } + } +#elif defined(JS_CODEGEN_ARM64) + MOZ_ASSERT(temp.isInvalid()); + if (access->type() == Scalar::Int64) { + masm.wasmStoreI64(*access, src.i64(), HeapReg, ptr); + } else { + masm.wasmStore(*access, src.any(), HeapReg, ptr); + } +#elif defined(JS_CODEGEN_LOONG64) + MOZ_ASSERT(temp.isInvalid()); + if (access->type() == Scalar::Int64) { + masm.wasmStoreI64(*access, src.i64(), HeapReg, ptr, ptr); + } else { + masm.wasmStore(*access, src.any(), HeapReg, ptr, ptr); + } +#else + MOZ_CRASH("BaseCompiler platform hook: store"); +#endif +} + +// ptr and src must not be the same register. +// This may destroy ptr and src. +void BaseCompiler::store(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI32 ptr, AnyReg src, RegI32 temp) { + prepareMemoryAccess(access, check, instance, ptr); + executeStore(access, check, instance, ptr, src, temp); +} + +#ifdef ENABLE_WASM_MEMORY64 +void BaseCompiler::store(MemoryAccessDesc* access, AccessCheck* check, + RegPtr instance, RegI64 ptr, AnyReg src, RegI64 temp) { + prepareMemoryAccess(access, check, instance, ptr); + // See comments in load() +# if !defined(JS_64BIT) + return executeStore(access, check, instance, RegI32(ptr.low), src, + maybeFromI64(temp)); +# elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + return executeStore(access, check, instance, RegI32(ptr.reg), src, + maybeFromI64(temp)); +# else + MOZ_CRASH("Missing platform hook"); +# endif +} +#endif + +template +void BaseCompiler::doLoadCommon(MemoryAccessDesc* access, AccessCheck check, + ValType type) { + RegPtr instance; + RegType temp; +#if defined(JS_CODEGEN_MIPS64) + temp = need(); +#endif + + switch (type.kind()) { + case ValType::I32: { + RegType rp = popMemoryAccess(access, &check); + RegI32 rv = needI32(); + instance = maybeLoadInstanceForAccess(check); + load(access, &check, instance, rp, AnyReg(rv), temp); + push(rv); + free(rp); + break; + } + case ValType::I64: { + RegI64 rv; + RegType rp; +#ifdef JS_CODEGEN_X86 + rv = specific_.abiReturnRegI64; + needI64(rv); + rp = popMemoryAccess(access, &check); +#else + rp = popMemoryAccess(access, &check); + rv = needI64(); +#endif + instance = maybeLoadInstanceForAccess(check); + load(access, &check, instance, rp, AnyReg(rv), temp); + push(rv); + free(rp); + break; + } + case ValType::F32: { + RegType rp = popMemoryAccess(access, &check); + RegF32 rv = needF32(); + instance = maybeLoadInstanceForAccess(check); + load(access, &check, instance, rp, AnyReg(rv), temp); + push(rv); + free(rp); + break; + } + case ValType::F64: { + RegType rp = popMemoryAccess(access, &check); + RegF64 rv = needF64(); + instance = maybeLoadInstanceForAccess(check); + load(access, &check, instance, rp, AnyReg(rv), temp); + push(rv); + free(rp); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegType rp = popMemoryAccess(access, &check); + RegV128 rv = needV128(); + instance = maybeLoadInstanceForAccess(check); + load(access, &check, instance, rp, AnyReg(rv), temp); + push(rv); + free(rp); + break; + } +#endif + default: + MOZ_CRASH("load type"); + break; + } + +#ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +#endif + maybeFree(temp); +} + +void BaseCompiler::loadCommon(MemoryAccessDesc* access, AccessCheck check, + ValType type) { + if (isMem32()) { + doLoadCommon(access, check, type); + } else { +#ifdef ENABLE_WASM_MEMORY64 + doLoadCommon(access, check, type); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } +} + +template +void BaseCompiler::doStoreCommon(MemoryAccessDesc* access, AccessCheck check, + ValType resultType) { + RegPtr instance; + RegType temp; +#if defined(JS_CODEGEN_MIPS64) + temp = need(); +#endif + + switch (resultType.kind()) { + case ValType::I32: { + RegI32 rv = popI32(); + RegType rp = popMemoryAccess(access, &check); + instance = maybeLoadInstanceForAccess(check); + store(access, &check, instance, rp, AnyReg(rv), temp); + free(rp); + free(rv); + break; + } + case ValType::I64: { + RegI64 rv = popI64(); + RegType rp = popMemoryAccess(access, &check); + instance = maybeLoadInstanceForAccess(check); + store(access, &check, instance, rp, AnyReg(rv), temp); + free(rp); + free(rv); + break; + } + case ValType::F32: { + RegF32 rv = popF32(); + RegType rp = popMemoryAccess(access, &check); + instance = maybeLoadInstanceForAccess(check); + store(access, &check, instance, rp, AnyReg(rv), temp); + free(rp); + free(rv); + break; + } + case ValType::F64: { + RegF64 rv = popF64(); + RegType rp = popMemoryAccess(access, &check); + instance = maybeLoadInstanceForAccess(check); + store(access, &check, instance, rp, AnyReg(rv), temp); + free(rp); + free(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 rv = popV128(); + RegType rp = popMemoryAccess(access, &check); + instance = maybeLoadInstanceForAccess(check); + store(access, &check, instance, rp, AnyReg(rv), temp); + free(rp); + free(rv); + break; + } +#endif + default: + MOZ_CRASH("store type"); + break; + } + +#ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +#endif + maybeFree(temp); +} + +void BaseCompiler::storeCommon(MemoryAccessDesc* access, AccessCheck check, + ValType type) { + if (isMem32()) { + doStoreCommon(access, check, type); + } else { +#ifdef ENABLE_WASM_MEMORY64 + doStoreCommon(access, check, type); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } +} + +// Convert something that may contain a heap index into a Register that can be +// used in an access. + +static inline Register ToRegister(RegI32 r) { return Register(r); } +#ifdef ENABLE_WASM_MEMORY64 +# ifdef JS_PUNBOX64 +static inline Register ToRegister(RegI64 r) { return r.reg; } +# else +static inline Register ToRegister(RegI64 r) { return r.low; } +# endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic operations. +// +// The atomic operations have very diverse per-platform needs for register +// allocation and temps. To handle that, the implementations are structured as +// a per-operation framework method that calls into platform-specific helpers +// (usually called PopAndAllocate, Perform, and Deallocate) in a per-operation +// namespace. This structure results in a little duplication and boilerplate +// but is otherwise clean and flexible and keeps code and supporting definitions +// entirely co-located. + +#ifdef WASM_HAS_HEAPREG + +// RegIndexType is RegI32 for Memory32 and RegI64 for Memory64. +template +BaseIndex BaseCompiler::prepareAtomicMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, + RegPtr instance, + RegIndexType ptr) { + MOZ_ASSERT(needInstanceForAccess(*check) == instance.isValid()); + prepareMemoryAccess(access, check, instance, ptr); + // At this point, 64-bit offsets will have been resolved. + return BaseIndex(HeapReg, ToRegister(ptr), TimesOne, access->offset()); +} + +#else + +// Some consumers depend on the returned Address not incorporating instance, as +// instance may be the scratch register. +// +// RegIndexType is RegI32 for Memory32 and RegI64 for Memory64. +template +Address BaseCompiler::prepareAtomicMemoryAccess(MemoryAccessDesc* access, + AccessCheck* check, + RegPtr instance, + RegIndexType ptr) { + MOZ_ASSERT(needInstanceForAccess(*check) == instance.isValid()); + prepareMemoryAccess(access, check, instance, ptr); + masm.addPtr(Address(instance, Instance::offsetOfMemoryBase()), + ToRegister(ptr)); + // At this point, 64-bit offsets will have been resolved. + return Address(ToRegister(ptr), access->offset()); +} + +#endif + +#ifndef WASM_HAS_HEAPREG +# ifdef JS_CODEGEN_X86 +using ScratchAtomicNoHeapReg = ScratchEBX; +# else +# error "Unimplemented porting interface" +# endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic load and store. + +namespace atomic_load64 { + +#ifdef JS_CODEGEN_ARM + +static void Allocate(BaseCompiler* bc, RegI64* rd, RegI64*) { + *rd = bc->needI64Pair(); +} + +static void Deallocate(BaseCompiler* bc, RegI64) {} + +#elif defined JS_CODEGEN_X86 + +static void Allocate(BaseCompiler* bc, RegI64* rd, RegI64* temp) { + // The result is in edx:eax, and we need ecx:ebx as a temp. But ebx will also + // be used as a scratch, so don't manage that here. + bc->needI32(bc->specific_.ecx); + *temp = bc->specific_.ecx_ebx; + bc->needI64(bc->specific_.edx_eax); + *rd = bc->specific_.edx_eax; +} + +static void Deallocate(BaseCompiler* bc, RegI64 temp) { + // See comment above. + MOZ_ASSERT(temp.high == js::jit::ecx); + bc->freeI32(bc->specific_.ecx); +} + +#elif defined(__wasi__) || (defined(JS_CODEGEN_NONE) && !defined(JS_64BIT)) + +static void Allocate(BaseCompiler*, RegI64*, RegI64*) {} +static void Deallocate(BaseCompiler*, RegI64) {} + +#endif + +} // namespace atomic_load64 + +#if !defined(JS_64BIT) +template +void BaseCompiler::atomicLoad64(MemoryAccessDesc* access) { + RegI64 rd, temp; + atomic_load64::Allocate(this, &rd, &temp); + + AccessCheck check; + RegIndexType rp = popMemoryAccess(access, &check); + +# ifdef WASM_HAS_HEAPREG + RegPtr instance = maybeLoadInstanceForAccess(check); + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + masm.wasmAtomicLoad64(*access, memaddr, temp, rd); +# ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +# endif +# else + ScratchAtomicNoHeapReg scratch(*this); + RegPtr instance = + maybeLoadInstanceForAccess(check, RegIntptrToRegPtr(scratch)); + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + masm.wasmAtomicLoad64(*access, memaddr, temp, rd); + MOZ_ASSERT(instance == scratch); +# endif + + free(rp); + atomic_load64::Deallocate(this, temp); + pushI64(rd); +} +#endif + +void BaseCompiler::atomicLoad(MemoryAccessDesc* access, ValType type) { + Scalar::Type viewType = access->type(); + if (Scalar::byteSize(viewType) <= sizeof(void*)) { + loadCommon(access, AccessCheck(), type); + return; + } + + MOZ_ASSERT(type == ValType::I64 && Scalar::byteSize(viewType) == 8); + +#if !defined(JS_64BIT) + if (isMem32()) { + atomicLoad64(access); + } else { +# ifdef ENABLE_WASM_MEMORY64 + atomicLoad64(access); +# else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +# endif + } +#else + MOZ_CRASH("Should not happen"); +#endif +} + +void BaseCompiler::atomicStore(MemoryAccessDesc* access, ValType type) { + Scalar::Type viewType = access->type(); + + if (Scalar::byteSize(viewType) <= sizeof(void*)) { + storeCommon(access, AccessCheck(), type); + return; + } + + MOZ_ASSERT(type == ValType::I64 && Scalar::byteSize(viewType) == 8); + +#if !defined(JS_64BIT) + if (isMem32()) { + atomicXchg64(access, WantResult(false)); + } else { +# ifdef ENABLE_WASM_MEMORY64 + atomicXchg64(access, WantResult(false)); +# else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +# endif + } +#else + MOZ_CRASH("Should not happen"); +#endif +} + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic RMW op= operations. + +void BaseCompiler::atomicRMW(MemoryAccessDesc* access, ValType type, + AtomicOp op) { + Scalar::Type viewType = access->type(); + if (Scalar::byteSize(viewType) <= 4) { + if (isMem32()) { + atomicRMW32(access, type, op); + } else { +#ifdef ENABLE_WASM_MEMORY64 + atomicRMW32(access, type, op); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + } else { + MOZ_ASSERT(type == ValType::I64 && Scalar::byteSize(viewType) == 8); + if (isMem32()) { + atomicRMW64(access, type, op); + } else { +#ifdef ENABLE_WASM_MEMORY64 + atomicRMW64(access, type, op); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + } +} + +namespace atomic_rmw32 { + +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) + +struct Temps { + // On x86 we use the ScratchI32 for the temp, otherwise we'd run out of + // registers for 64-bit operations. +# if defined(JS_CODEGEN_X64) + RegI32 t0; +# endif +}; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, AtomicOp op, RegI32* rd, + RegI32* rv, Temps* temps) { + bc->needI32(bc->specific_.eax); + if (op == AtomicFetchAddOp || op == AtomicFetchSubOp) { + // We use xadd, so source and destination are the same. Using + // eax here is overconstraining, but for byte operations on x86 + // we do need something with a byte register. + if (type == ValType::I64) { + *rv = bc->popI64ToSpecificI32(bc->specific_.eax); + } else { + *rv = bc->popI32ToSpecific(bc->specific_.eax); + } + *rd = *rv; + } else { + // We use a cmpxchg loop. The output must be eax; the input + // must be in a separate register since it may be used several + // times. + if (type == ValType::I64) { + *rv = bc->popI64ToI32(); + } else { + *rv = bc->popI32(); + } + *rd = bc->specific_.eax; +# ifdef JS_CODEGEN_X64 + temps->t0 = bc->needI32(); +# endif + } +} + +template +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, T srcAddr, + AtomicOp op, RegI32 rv, RegI32 rd, const Temps& temps) { +# ifdef JS_CODEGEN_X64 + RegI32 temp = temps.t0; +# else + RegI32 temp; + ScratchI32 scratch(*bc); + if (op != AtomicFetchAddOp && op != AtomicFetchSubOp) { + temp = scratch; + } +# endif + bc->masm.wasmAtomicFetchOp(access, op, rv, srcAddr, temp, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rv, const Temps& temps) { + if (rv != bc->specific_.eax) { + bc->freeI32(rv); + } +# ifdef JS_CODEGEN_X64 + bc->maybeFree(temps.t0); +# endif +} + +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + +struct Temps { + RegI32 t0; +}; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, AtomicOp op, RegI32* rd, + RegI32* rv, Temps* temps) { + *rv = type == ValType::I64 ? bc->popI64ToI32() : bc->popI32(); + temps->t0 = bc->needI32(); + *rd = bc->needI32(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, AtomicOp op, RegI32 rv, RegI32 rd, + const Temps& temps) { + bc->masm.wasmAtomicFetchOp(access, op, rv, srcAddr, temps.t0, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rv, const Temps& temps) { + bc->freeI32(rv); + bc->freeI32(temps.t0); +} + +#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + +struct Temps { + RegI32 t0, t1, t2; +}; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, AtomicOp op, RegI32* rd, + RegI32* rv, Temps* temps) { + *rv = type == ValType::I64 ? bc->popI64ToI32() : bc->popI32(); + if (Scalar::byteSize(viewType) < 4) { + temps->t0 = bc->needI32(); + temps->t1 = bc->needI32(); + temps->t2 = bc->needI32(); + } + *rd = bc->needI32(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, AtomicOp op, RegI32 rv, RegI32 rd, + const Temps& temps) { + bc->masm.wasmAtomicFetchOp(access, op, rv, srcAddr, temps.t0, temps.t1, + temps.t2, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rv, const Temps& temps) { + bc->freeI32(rv); + bc->maybeFree(temps.t0); + bc->maybeFree(temps.t1); + bc->maybeFree(temps.t2); +} + +#elif defined(JS_CODEGEN_RISCV64) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler*, ValType, Scalar::Type, AtomicOp, + RegI32*, RegI32*, Temps*) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, AtomicOp, + RegI32, RegI32, const Temps&) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +static void Deallocate(BaseCompiler*, RegI32, const Temps&) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler*, ValType, Scalar::Type, AtomicOp, + RegI32*, RegI32*, Temps*) {} + +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, AtomicOp, + RegI32, RegI32, const Temps&) {} + +static void Deallocate(BaseCompiler*, RegI32, const Temps&) {} + +#endif + +} // namespace atomic_rmw32 + +template +void BaseCompiler::atomicRMW32(MemoryAccessDesc* access, ValType type, + AtomicOp op) { + Scalar::Type viewType = access->type(); + RegI32 rd, rv; + atomic_rmw32::Temps temps; + atomic_rmw32::PopAndAllocate(this, type, viewType, op, &rd, &rv, &temps); + + AccessCheck check; + RegIndexType rp = popMemoryAccess(access, &check); + RegPtr instance = maybeLoadInstanceForAccess(check); + + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_rmw32::Perform(this, *access, memaddr, op, rv, rd, temps); + +#ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +#endif + atomic_rmw32::Deallocate(this, rv, temps); + free(rp); + + if (type == ValType::I64) { + pushU32AsI64(rd); + } else { + pushI32(rd); + } +} + +namespace atomic_rmw64 { + +#if defined(JS_CODEGEN_X64) + +static void PopAndAllocate(BaseCompiler* bc, AtomicOp op, RegI64* rd, + RegI64* rv, RegI64* temp) { + if (op == AtomicFetchAddOp || op == AtomicFetchSubOp) { + // We use xaddq, so input and output must be the same register. + *rv = bc->popI64(); + *rd = *rv; + } else { + // We use a cmpxchgq loop, so the output must be rax and we need a temp. + bc->needI64(bc->specific_.rax); + *rd = bc->specific_.rax; + *rv = bc->popI64(); + *temp = bc->needI64(); + } +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, AtomicOp op, RegI64 rv, RegI64 temp, + RegI64 rd) { + bc->masm.wasmAtomicFetchOp64(access, op, rv, srcAddr, temp, rd); +} + +static void Deallocate(BaseCompiler* bc, AtomicOp op, RegI64 rv, RegI64 temp) { + bc->maybeFree(temp); + if (op != AtomicFetchAddOp && op != AtomicFetchSubOp) { + bc->freeI64(rv); + } +} + +#elif defined(JS_CODEGEN_X86) + +// Register allocation is tricky, see comments at atomic_xchg64 below. +// +// - Initially rv=ecx:edx and eax is reserved, rd=unallocated. +// - Then rp is popped into esi+edi because those are the only available. +// - The Setup operation makes rd=edx:eax. +// - Deallocation then frees only the ecx part of rv. +// +// The temp is unused here. + +static void PopAndAllocate(BaseCompiler* bc, AtomicOp op, RegI64* rd, + RegI64* rv, RegI64*) { + bc->needI32(bc->specific_.eax); + bc->needI32(bc->specific_.ecx); + bc->needI32(bc->specific_.edx); + *rv = RegI64(Register64(bc->specific_.ecx, bc->specific_.edx)); + bc->popI64ToSpecific(*rv); +} + +static void Setup(BaseCompiler* bc, RegI64* rd) { *rd = bc->specific_.edx_eax; } + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + Address srcAddr, AtomicOp op, RegI64 rv, RegI64, RegI64 rd, + const ScratchAtomicNoHeapReg& scratch) { + MOZ_ASSERT(rv.high == bc->specific_.ecx); + MOZ_ASSERT(Register(scratch) == js::jit::ebx); + + bc->fr.pushGPR(rv.high); + bc->fr.pushGPR(rv.low); + Address value(StackPointer, 0); + + bc->masm.wasmAtomicFetchOp64(access, op, value, srcAddr, + bc->specific_.ecx_ebx, rd); + + bc->fr.popBytes(8); +} + +static void Deallocate(BaseCompiler* bc, AtomicOp, RegI64, RegI64) { + bc->freeI32(bc->specific_.ecx); +} + +#elif defined(JS_CODEGEN_ARM) + +static void PopAndAllocate(BaseCompiler* bc, AtomicOp op, RegI64* rd, + RegI64* rv, RegI64* temp) { + // We use a ldrex/strexd loop so the temp and the output must be + // odd/even pairs. + *rv = bc->popI64(); + *temp = bc->needI64Pair(); + *rd = bc->needI64Pair(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, AtomicOp op, RegI64 rv, RegI64 temp, + RegI64 rd) { + bc->masm.wasmAtomicFetchOp64(access, op, rv, srcAddr, temp, rd); +} + +static void Deallocate(BaseCompiler* bc, AtomicOp op, RegI64 rv, RegI64 temp) { + bc->freeI64(rv); + bc->freeI64(temp); +} + +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) + +static void PopAndAllocate(BaseCompiler* bc, AtomicOp op, RegI64* rd, + RegI64* rv, RegI64* temp) { + *rv = bc->popI64(); + *temp = bc->needI64(); + *rd = bc->needI64(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, AtomicOp op, RegI64 rv, RegI64 temp, + RegI64 rd) { + bc->masm.wasmAtomicFetchOp64(access, op, rv, srcAddr, temp, rd); +} + +static void Deallocate(BaseCompiler* bc, AtomicOp op, RegI64 rv, RegI64 temp) { + bc->freeI64(rv); + bc->freeI64(temp); +} +#elif defined(JS_CODEGEN_RISCV64) + +static void PopAndAllocate(BaseCompiler*, AtomicOp, RegI64*, RegI64*, RegI64*) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, + AtomicOp op, RegI64, RegI64, RegI64) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +static void Deallocate(BaseCompiler*, AtomicOp, RegI64, RegI64) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + +static void PopAndAllocate(BaseCompiler*, AtomicOp, RegI64*, RegI64*, RegI64*) { +} + +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, + AtomicOp op, RegI64, RegI64, RegI64) {} + +static void Deallocate(BaseCompiler*, AtomicOp, RegI64, RegI64) {} + +#endif + +} // namespace atomic_rmw64 + +template +void BaseCompiler::atomicRMW64(MemoryAccessDesc* access, ValType type, + AtomicOp op) { + RegI64 rd, rv, temp; + atomic_rmw64::PopAndAllocate(this, op, &rd, &rv, &temp); + + AccessCheck check; + RegIndexType rp = popMemoryAccess(access, &check); + +#if defined(WASM_HAS_HEAPREG) + RegPtr instance = maybeLoadInstanceForAccess(check); + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_rmw64::Perform(this, *access, memaddr, op, rv, temp, rd); +# ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +# endif +#else + ScratchAtomicNoHeapReg scratch(*this); + RegPtr instance = + maybeLoadInstanceForAccess(check, RegIntptrToRegPtr(scratch)); + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_rmw64::Setup(this, &rd); + atomic_rmw64::Perform(this, *access, memaddr, op, rv, temp, rd, scratch); + MOZ_ASSERT(instance == scratch); +#endif + + free(rp); + atomic_rmw64::Deallocate(this, op, rv, temp); + + pushI64(rd); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic exchange (also used for atomic store in some cases). + +void BaseCompiler::atomicXchg(MemoryAccessDesc* access, ValType type) { + Scalar::Type viewType = access->type(); + if (Scalar::byteSize(viewType) <= 4) { + if (isMem32()) { + atomicXchg32(access, type); + } else { +#ifdef ENABLE_WASM_MEMORY64 + atomicXchg32(access, type); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + } else { + MOZ_ASSERT(type == ValType::I64 && Scalar::byteSize(viewType) == 8); + if (isMem32()) { + atomicXchg64(access, WantResult(true)); + } else { +#ifdef ENABLE_WASM_MEMORY64 + atomicXchg64(access, WantResult(true)); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + } +} + +namespace atomic_xchg32 { + +#if defined(JS_CODEGEN_X64) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rd, RegI32* rv, + Temps*) { + // The xchg instruction reuses rv as rd. + *rv = (type == ValType::I64) ? bc->popI64ToI32() : bc->popI32(); + *rd = *rv; +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI32 rv, RegI32 rd, const Temps&) { + bc->masm.wasmAtomicExchange(access, srcAddr, rv, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32, const Temps&) {} + +#elif defined(JS_CODEGEN_X86) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rd, RegI32* rv, + Temps*) { + // The xchg instruction reuses rv as rd. + *rv = (type == ValType::I64) ? bc->popI64ToI32() : bc->popI32(); + *rd = *rv; +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + Address srcAddr, RegI32 rv, RegI32 rd, const Temps&) { + if (access.type() == Scalar::Uint8 && !bc->ra.isSingleByteI32(rd)) { + ScratchI8 scratch(*bc); + // The output register must have a byte persona. + bc->masm.wasmAtomicExchange(access, srcAddr, rv, scratch); + bc->masm.movl(scratch, rd); + } else { + bc->masm.wasmAtomicExchange(access, srcAddr, rv, rd); + } +} + +static void Deallocate(BaseCompiler* bc, RegI32, const Temps&) {} + +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rd, RegI32* rv, + Temps*) { + *rv = (type == ValType::I64) ? bc->popI64ToI32() : bc->popI32(); + *rd = bc->needI32(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI32 rv, RegI32 rd, const Temps&) { + bc->masm.wasmAtomicExchange(access, srcAddr, rv, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rv, const Temps&) { + bc->freeI32(rv); +} + +#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + +struct Temps { + RegI32 t0, t1, t2; +}; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rd, RegI32* rv, + Temps* temps) { + *rv = (type == ValType::I64) ? bc->popI64ToI32() : bc->popI32(); + if (Scalar::byteSize(viewType) < 4) { + temps->t0 = bc->needI32(); + temps->t1 = bc->needI32(); + temps->t2 = bc->needI32(); + } + *rd = bc->needI32(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI32 rv, RegI32 rd, + const Temps& temps) { + bc->masm.wasmAtomicExchange(access, srcAddr, rv, temps.t0, temps.t1, temps.t2, + rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rv, const Temps& temps) { + bc->freeI32(rv); + bc->maybeFree(temps.t0); + bc->maybeFree(temps.t1); + bc->maybeFree(temps.t2); +} + +#elif defined(JS_CODEGEN_RISCV64) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler*, ValType, Scalar::Type, RegI32*, + RegI32*, Temps*) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, RegI32, + RegI32, const Temps&) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} +static void Deallocate(BaseCompiler*, RegI32, const Temps&) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler*, ValType, Scalar::Type, RegI32*, + RegI32*, Temps*) {} +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, RegI32, + RegI32, const Temps&) {} +static void Deallocate(BaseCompiler*, RegI32, const Temps&) {} + +#endif + +} // namespace atomic_xchg32 + +template +void BaseCompiler::atomicXchg32(MemoryAccessDesc* access, ValType type) { + Scalar::Type viewType = access->type(); + + RegI32 rd, rv; + atomic_xchg32::Temps temps; + atomic_xchg32::PopAndAllocate(this, type, viewType, &rd, &rv, &temps); + + AccessCheck check; + + RegIndexType rp = popMemoryAccess(access, &check); + RegPtr instance = maybeLoadInstanceForAccess(check); + + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_xchg32::Perform(this, *access, memaddr, rv, rd, temps); + +#ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +#endif + free(rp); + atomic_xchg32::Deallocate(this, rv, temps); + + if (type == ValType::I64) { + pushU32AsI64(rd); + } else { + pushI32(rd); + } +} + +namespace atomic_xchg64 { + +#if defined(JS_CODEGEN_X64) + +static void PopAndAllocate(BaseCompiler* bc, RegI64* rd, RegI64* rv) { + *rv = bc->popI64(); + *rd = *rv; +} + +static void Deallocate(BaseCompiler* bc, RegI64 rd, RegI64) { + bc->maybeFree(rd); +} + +#elif defined(JS_CODEGEN_X86) + +// Register allocation is tricky in several ways. +// +// - For a 64-bit access on memory64 we need six registers for rd, rv, and rp, +// but have only five (as the temp ebx is needed too), so we target all +// registers explicitly to make sure there's space. +// +// - We'll be using cmpxchg8b, and when we do the operation, rv must be in +// ecx:ebx, and rd must be edx:eax. We can't use ebx for rv initially because +// we need ebx for a scratch also, so use a separate temp and move the value +// to ebx just before the operation. +// +// In sum: +// +// - Initially rv=ecx:edx and eax is reserved, rd=unallocated. +// - Then rp is popped into esi+edi because those are the only available. +// - The Setup operation makes rv=ecx:ebx and rd=edx:eax and moves edx->ebx. +// - Deallocation then frees only the ecx part of rv. + +static void PopAndAllocate(BaseCompiler* bc, RegI64* rd, RegI64* rv) { + bc->needI32(bc->specific_.ecx); + bc->needI32(bc->specific_.edx); + bc->needI32(bc->specific_.eax); + *rv = RegI64(Register64(bc->specific_.ecx, bc->specific_.edx)); + bc->popI64ToSpecific(*rv); +} + +static void Setup(BaseCompiler* bc, RegI64* rv, RegI64* rd, + const ScratchAtomicNoHeapReg& scratch) { + MOZ_ASSERT(rv->high == bc->specific_.ecx); + MOZ_ASSERT(Register(scratch) == js::jit::ebx); + bc->masm.move32(rv->low, scratch); + *rv = bc->specific_.ecx_ebx; + *rd = bc->specific_.edx_eax; +} + +static void Deallocate(BaseCompiler* bc, RegI64 rd, RegI64 rv) { + MOZ_ASSERT(rd == bc->specific_.edx_eax || rd == RegI64::Invalid()); + bc->maybeFree(rd); + bc->freeI32(bc->specific_.ecx); +} + +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) + +static void PopAndAllocate(BaseCompiler* bc, RegI64* rd, RegI64* rv) { + *rv = bc->popI64(); + *rd = bc->needI64(); +} + +static void Deallocate(BaseCompiler* bc, RegI64 rd, RegI64 rv) { + bc->freeI64(rv); + bc->maybeFree(rd); +} + +#elif defined(JS_CODEGEN_ARM) + +static void PopAndAllocate(BaseCompiler* bc, RegI64* rd, RegI64* rv) { + // Both rv and rd must be odd/even pairs. + *rv = bc->popI64ToSpecific(bc->needI64Pair()); + *rd = bc->needI64Pair(); +} + +static void Deallocate(BaseCompiler* bc, RegI64 rd, RegI64 rv) { + bc->freeI64(rv); + bc->maybeFree(rd); +} + +#elif defined(JS_CODEGEN_RISCV64) + +static void PopAndAllocate(BaseCompiler*, RegI64*, RegI64*) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} +static void Deallocate(BaseCompiler*, RegI64, RegI64) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + +static void PopAndAllocate(BaseCompiler*, RegI64*, RegI64*) {} +static void Deallocate(BaseCompiler*, RegI64, RegI64) {} + +#endif + +} // namespace atomic_xchg64 + +template +void BaseCompiler::atomicXchg64(MemoryAccessDesc* access, + WantResult wantResult) { + RegI64 rd, rv; + atomic_xchg64::PopAndAllocate(this, &rd, &rv); + + AccessCheck check; + RegIndexType rp = popMemoryAccess(access, &check); + +#ifdef WASM_HAS_HEAPREG + RegPtr instance = maybeLoadInstanceForAccess(check); + auto memaddr = + prepareAtomicMemoryAccess(access, &check, instance, rp); + masm.wasmAtomicExchange64(*access, memaddr, rv, rd); +# ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +# endif +#else + ScratchAtomicNoHeapReg scratch(*this); + RegPtr instance = + maybeLoadInstanceForAccess(check, RegIntptrToRegPtr(scratch)); + Address memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_xchg64::Setup(this, &rv, &rd, scratch); + masm.wasmAtomicExchange64(*access, memaddr, rv, rd); + MOZ_ASSERT(instance == scratch); +#endif + + free(rp); + if (wantResult) { + pushI64(rd); + rd = RegI64::Invalid(); + } + atomic_xchg64::Deallocate(this, rd, rv); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic compare-exchange. + +void BaseCompiler::atomicCmpXchg(MemoryAccessDesc* access, ValType type) { + Scalar::Type viewType = access->type(); + if (Scalar::byteSize(viewType) <= 4) { + if (isMem32()) { + atomicCmpXchg32(access, type); + } else { +#ifdef ENABLE_WASM_MEMORY64 + atomicCmpXchg32(access, type); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + } else { + MOZ_ASSERT(type == ValType::I64 && Scalar::byteSize(viewType) == 8); + if (isMem32()) { + atomicCmpXchg64(access, type); + } else { +#ifdef ENABLE_WASM_MEMORY64 + atomicCmpXchg64(access, type); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + } +} + +namespace atomic_cmpxchg32 { + +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rexpect, RegI32* rnew, + RegI32* rd, Temps*) { + // For cmpxchg, the expected value and the result are both in eax. + bc->needI32(bc->specific_.eax); + if (type == ValType::I64) { + *rnew = bc->popI64ToI32(); + *rexpect = bc->popI64ToSpecificI32(bc->specific_.eax); + } else { + *rnew = bc->popI32(); + *rexpect = bc->popI32ToSpecific(bc->specific_.eax); + } + *rd = *rexpect; +} + +template +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, T srcAddr, + RegI32 rexpect, RegI32 rnew, RegI32 rd, const Temps&) { +# if defined(JS_CODEGEN_X86) + ScratchI8 scratch(*bc); + if (access.type() == Scalar::Uint8) { + MOZ_ASSERT(rd == bc->specific_.eax); + if (!bc->ra.isSingleByteI32(rnew)) { + // The replacement value must have a byte persona. + bc->masm.movl(rnew, scratch); + rnew = scratch; + } + } +# endif + bc->masm.wasmCompareExchange(access, srcAddr, rexpect, rnew, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32, RegI32 rnew, const Temps&) { + bc->freeI32(rnew); +} + +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rexpect, RegI32* rnew, + RegI32* rd, Temps*) { + if (type == ValType::I64) { + *rnew = bc->popI64ToI32(); + *rexpect = bc->popI64ToI32(); + } else { + *rnew = bc->popI32(); + *rexpect = bc->popI32(); + } + *rd = bc->needI32(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI32 rexpect, RegI32 rnew, RegI32 rd, + const Temps&) { + bc->masm.wasmCompareExchange(access, srcAddr, rexpect, rnew, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rexpect, RegI32 rnew, + const Temps&) { + bc->freeI32(rnew); + bc->freeI32(rexpect); +} + +#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + +struct Temps { + RegI32 t0, t1, t2; +}; + +static void PopAndAllocate(BaseCompiler* bc, ValType type, + Scalar::Type viewType, RegI32* rexpect, RegI32* rnew, + RegI32* rd, Temps* temps) { + if (type == ValType::I64) { + *rnew = bc->popI64ToI32(); + *rexpect = bc->popI64ToI32(); + } else { + *rnew = bc->popI32(); + *rexpect = bc->popI32(); + } + if (Scalar::byteSize(viewType) < 4) { + temps->t0 = bc->needI32(); + temps->t1 = bc->needI32(); + temps->t2 = bc->needI32(); + } + *rd = bc->needI32(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI32 rexpect, RegI32 rnew, RegI32 rd, + const Temps& temps) { + bc->masm.wasmCompareExchange(access, srcAddr, rexpect, rnew, temps.t0, + temps.t1, temps.t2, rd); +} + +static void Deallocate(BaseCompiler* bc, RegI32 rexpect, RegI32 rnew, + const Temps& temps) { + bc->freeI32(rnew); + bc->freeI32(rexpect); + bc->maybeFree(temps.t0); + bc->maybeFree(temps.t1); + bc->maybeFree(temps.t2); +} + +#elif defined(JS_CODEGEN_RISCV64) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler*, ValType, Scalar::Type, RegI32*, + RegI32*, RegI32*, Temps*) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, RegI32, + RegI32, RegI32, const Temps& temps) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +static void Deallocate(BaseCompiler*, RegI32, RegI32, const Temps&) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + +using Temps = Nothing; + +static void PopAndAllocate(BaseCompiler*, ValType, Scalar::Type, RegI32*, + RegI32*, RegI32*, Temps*) {} + +static void Perform(BaseCompiler*, const MemoryAccessDesc&, BaseIndex, RegI32, + RegI32, RegI32, const Temps& temps) {} + +static void Deallocate(BaseCompiler*, RegI32, RegI32, const Temps&) {} + +#endif + +} // namespace atomic_cmpxchg32 + +template +void BaseCompiler::atomicCmpXchg32(MemoryAccessDesc* access, ValType type) { + Scalar::Type viewType = access->type(); + RegI32 rexpect, rnew, rd; + atomic_cmpxchg32::Temps temps; + atomic_cmpxchg32::PopAndAllocate(this, type, viewType, &rexpect, &rnew, &rd, + &temps); + + AccessCheck check; + RegIndexType rp = popMemoryAccess(access, &check); + RegPtr instance = maybeLoadInstanceForAccess(check); + + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_cmpxchg32::Perform(this, *access, memaddr, rexpect, rnew, rd, temps); + +#ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +#endif + free(rp); + atomic_cmpxchg32::Deallocate(this, rexpect, rnew, temps); + + if (type == ValType::I64) { + pushU32AsI64(rd); + } else { + pushI32(rd); + } +} + +namespace atomic_cmpxchg64 { + +// The templates are needed for x86 code generation, which needs complicated +// register allocation for memory64. + +template +static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd); + +template +static void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew); + +#if defined(JS_CODEGEN_X64) + +template +static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) { + // For cmpxchg, the expected value and the result are both in rax. + bc->needI64(bc->specific_.rax); + *rnew = bc->popI64(); + *rexpect = bc->popI64ToSpecific(bc->specific_.rax); + *rd = *rexpect; +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd) { + bc->masm.wasmCompareExchange64(access, srcAddr, rexpect, rnew, rd); +} + +template +static void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) { + bc->freeI64(rnew); +} + +#elif defined(JS_CODEGEN_X86) + +template +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + Address srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd, + ScratchAtomicNoHeapReg& scratch); + +// Memory32: For cmpxchg8b, the expected value and the result are both in +// edx:eax, and the replacement value is in ecx:ebx. But we can't allocate ebx +// initially because we need it later for a scratch, so instead we allocate a +// temp to hold the low word of 'new'. + +template <> +void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) { + bc->needI64(bc->specific_.edx_eax); + bc->needI32(bc->specific_.ecx); + RegI32 tmp = bc->needI32(); + *rnew = bc->popI64ToSpecific(RegI64(Register64(bc->specific_.ecx, tmp))); + *rexpect = bc->popI64ToSpecific(bc->specific_.edx_eax); + *rd = *rexpect; +} + +template <> +void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + Address srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd, + ScratchAtomicNoHeapReg& scratch) { + MOZ_ASSERT(Register(scratch) == js::jit::ebx); + MOZ_ASSERT(rnew.high == bc->specific_.ecx); + bc->masm.move32(rnew.low, ebx); + bc->masm.wasmCompareExchange64(access, srcAddr, rexpect, + bc->specific_.ecx_ebx, rd); +} + +template <> +void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) { + bc->freeI64(rnew); +} + +// Memory64: Register allocation is particularly hairy here. With memory64, we +// have up to seven live values: i64 expected-value, i64 new-value, i64 pointer, +// and instance. The instance can use the scratch but there's no avoiding that +// we'll run out of registers. +// +// Unlike for the rmw ops, we can't use edx as the rnew.low since it's used +// for the rexpect.high. And we can't push anything onto the stack while we're +// popping the memory address because the memory address may be on the stack. + +# ifdef ENABLE_WASM_MEMORY64 +template <> +void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) { + // We reserve these (and ebx). The 64-bit pointer will end up in esi+edi. + bc->needI32(bc->specific_.eax); + bc->needI32(bc->specific_.ecx); + bc->needI32(bc->specific_.edx); + + // Pop the 'new' value and stash it in the instance scratch area. Do not + // initialize *rnew to anything. + RegI64 tmp(Register64(bc->specific_.ecx, bc->specific_.edx)); + bc->popI64ToSpecific(tmp); + { + ScratchPtr instanceScratch(*bc); + bc->stashI64(instanceScratch, tmp); + } + + *rexpect = bc->popI64ToSpecific(bc->specific_.edx_eax); + *rd = *rexpect; +} + +template <> +void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + Address srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd, + ScratchAtomicNoHeapReg& scratch) { + MOZ_ASSERT(rnew.isInvalid()); + rnew = bc->specific_.ecx_ebx; + + bc->unstashI64(RegPtr(Register(bc->specific_.ecx)), rnew); + bc->masm.wasmCompareExchange64(access, srcAddr, rexpect, rnew, rd); +} + +template <> +void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) { + // edx:ebx have been pushed as the result, and the pointer was freed + // separately in the caller, so just free ecx. + bc->free(bc->specific_.ecx); +} +# endif + +#elif defined(JS_CODEGEN_ARM) + +template +static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) { + // The replacement value and the result must both be odd/even pairs. + *rnew = bc->popI64Pair(); + *rexpect = bc->popI64(); + *rd = bc->needI64Pair(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd) { + bc->masm.wasmCompareExchange64(access, srcAddr, rexpect, rnew, rd); +} + +template +static void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) { + bc->freeI64(rexpect); + bc->freeI64(rnew); +} + +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) + +template +static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) { + *rnew = bc->popI64(); + *rexpect = bc->popI64(); + *rd = bc->needI64(); +} + +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd) { + bc->masm.wasmCompareExchange64(access, srcAddr, rexpect, rnew, rd); +} + +template +static void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) { + bc->freeI64(rexpect); + bc->freeI64(rnew); +} + +#elif defined(JS_CODEGEN_RISCV64) + +template +static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} +template +static void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) { + MOZ_CRASH("UNIMPLEMENTED ON RISCV64"); +} + +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + +template +static void PopAndAllocate(BaseCompiler* bc, RegI64* rexpect, RegI64* rnew, + RegI64* rd) {} +static void Perform(BaseCompiler* bc, const MemoryAccessDesc& access, + BaseIndex srcAddr, RegI64 rexpect, RegI64 rnew, RegI64 rd) { +} +template +static void Deallocate(BaseCompiler* bc, RegI64 rexpect, RegI64 rnew) {} + +#endif + +} // namespace atomic_cmpxchg64 + +template +void BaseCompiler::atomicCmpXchg64(MemoryAccessDesc* access, ValType type) { + RegI64 rexpect, rnew, rd; + atomic_cmpxchg64::PopAndAllocate(this, &rexpect, &rnew, &rd); + + AccessCheck check; + RegIndexType rp = popMemoryAccess(access, &check); + +#ifdef WASM_HAS_HEAPREG + RegPtr instance = maybeLoadInstanceForAccess(check); + auto memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_cmpxchg64::Perform(this, *access, memaddr, rexpect, rnew, rd); +# ifndef RABALDR_PIN_INSTANCE + maybeFree(instance); +# endif +#else + ScratchAtomicNoHeapReg scratch(*this); + RegPtr instance = + maybeLoadInstanceForAccess(check, RegIntptrToRegPtr(scratch)); + Address memaddr = prepareAtomicMemoryAccess(access, &check, instance, rp); + atomic_cmpxchg64::Perform(this, *access, memaddr, rexpect, rnew, + rd, scratch); + MOZ_ASSERT(instance == scratch); +#endif + + free(rp); + atomic_cmpxchg64::Deallocate(this, rexpect, rnew); + + pushI64(rd); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Synchronization. + +bool BaseCompiler::atomicWait(ValType type, MemoryAccessDesc* access) { + switch (type.kind()) { + case ValType::I32: { + RegI64 timeout = popI64(); + RegI32 val = popI32(); + + if (isMem32()) { + computeEffectiveAddress(access); + } else { +#ifdef ENABLE_WASM_MEMORY64 + computeEffectiveAddress(access); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + + pushI32(val); + pushI64(timeout); + + if (!emitInstanceCall(isMem32() ? SASigWaitI32M32 : SASigWaitI32M64)) { + return false; + } + break; + } + case ValType::I64: { + RegI64 timeout = popI64(); + RegI64 val = popI64(); + + if (isMem32()) { + computeEffectiveAddress(access); + } else { +#ifdef ENABLE_WASM_MEMORY64 +# ifdef JS_CODEGEN_X86 + { + ScratchPtr scratch(*this); + stashI64(scratch, val); + freeI64(val); + } +# endif + computeEffectiveAddress(access); +# ifdef JS_CODEGEN_X86 + { + ScratchPtr scratch(*this); + val = needI64(); + unstashI64(scratch, val); + } +# endif +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + + pushI64(val); + pushI64(timeout); + + if (!emitInstanceCall(isMem32() ? SASigWaitI64M32 : SASigWaitI64M64)) { + return false; + } + break; + } + default: + MOZ_CRASH(); + } + + return true; +} + +bool BaseCompiler::atomicWake(MemoryAccessDesc* access) { + RegI32 count = popI32(); + + if (isMem32()) { + computeEffectiveAddress(access); + } else { +#ifdef ENABLE_WASM_MEMORY64 + computeEffectiveAddress(access); +#else + MOZ_CRASH("Memory64 not enabled / supported on this platform"); +#endif + } + + pushI32(count); + return emitInstanceCall(isMem32() ? SASigWakeM32 : SASigWakeM64); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk memory. + +void BaseCompiler::memCopyInlineM32() { + MOZ_ASSERT(MaxInlineMemoryCopyLength != 0); + + int32_t signedLength; + MOZ_ALWAYS_TRUE(popConst(&signedLength)); + uint32_t length = signedLength; + MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryCopyLength); + + RegI32 src = popI32(); + RegI32 dest = popI32(); + + // Compute the number of copies of each width we will need to do + size_t remainder = length; +#ifdef ENABLE_WASM_SIMD + size_t numCopies16 = 0; + if (MacroAssembler::SupportsFastUnalignedFPAccesses()) { + numCopies16 = remainder / sizeof(V128); + remainder %= sizeof(V128); + } +#endif +#ifdef JS_64BIT + size_t numCopies8 = remainder / sizeof(uint64_t); + remainder %= sizeof(uint64_t); +#endif + size_t numCopies4 = remainder / sizeof(uint32_t); + remainder %= sizeof(uint32_t); + size_t numCopies2 = remainder / sizeof(uint16_t); + remainder %= sizeof(uint16_t); + size_t numCopies1 = remainder; + + // Load all source bytes onto the value stack from low to high using the + // widest transfer width we can for the system. We will trap without writing + // anything if any source byte is out-of-bounds. + bool omitBoundsCheck = false; + size_t offset = 0; + +#ifdef ENABLE_WASM_SIMD + for (uint32_t i = 0; i < numCopies16; i++) { + RegI32 temp = needI32(); + moveI32(src, temp); + pushI32(temp); + + MemoryAccessDesc access(Scalar::Simd128, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + loadCommon(&access, check, ValType::V128); + + offset += sizeof(V128); + omitBoundsCheck = true; + } +#endif + +#ifdef JS_64BIT + for (uint32_t i = 0; i < numCopies8; i++) { + RegI32 temp = needI32(); + moveI32(src, temp); + pushI32(temp); + + MemoryAccessDesc access(Scalar::Int64, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + loadCommon(&access, check, ValType::I64); + + offset += sizeof(uint64_t); + omitBoundsCheck = true; + } +#endif + + for (uint32_t i = 0; i < numCopies4; i++) { + RegI32 temp = needI32(); + moveI32(src, temp); + pushI32(temp); + + MemoryAccessDesc access(Scalar::Uint32, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + loadCommon(&access, check, ValType::I32); + + offset += sizeof(uint32_t); + omitBoundsCheck = true; + } + + if (numCopies2) { + RegI32 temp = needI32(); + moveI32(src, temp); + pushI32(temp); + + MemoryAccessDesc access(Scalar::Uint16, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + loadCommon(&access, check, ValType::I32); + + offset += sizeof(uint16_t); + omitBoundsCheck = true; + } + + if (numCopies1) { + RegI32 temp = needI32(); + moveI32(src, temp); + pushI32(temp); + + MemoryAccessDesc access(Scalar::Uint8, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + loadCommon(&access, check, ValType::I32); + } + + // Store all source bytes from the value stack to the destination from + // high to low. We will trap without writing anything on the first store + // if any dest byte is out-of-bounds. + offset = length; + omitBoundsCheck = false; + + if (numCopies1) { + offset -= sizeof(uint8_t); + + RegI32 value = popI32(); + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI32(value); + + MemoryAccessDesc access(Scalar::Uint8, 1, offset, bytecodeOffset()); + AccessCheck check; + storeCommon(&access, check, ValType::I32); + + omitBoundsCheck = true; + } + + if (numCopies2) { + offset -= sizeof(uint16_t); + + RegI32 value = popI32(); + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI32(value); + + MemoryAccessDesc access(Scalar::Uint16, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::I32); + + omitBoundsCheck = true; + } + + for (uint32_t i = 0; i < numCopies4; i++) { + offset -= sizeof(uint32_t); + + RegI32 value = popI32(); + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI32(value); + + MemoryAccessDesc access(Scalar::Uint32, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::I32); + + omitBoundsCheck = true; + } + +#ifdef JS_64BIT + for (uint32_t i = 0; i < numCopies8; i++) { + offset -= sizeof(uint64_t); + + RegI64 value = popI64(); + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI64(value); + + MemoryAccessDesc access(Scalar::Int64, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::I64); + + omitBoundsCheck = true; + } +#endif + +#ifdef ENABLE_WASM_SIMD + for (uint32_t i = 0; i < numCopies16; i++) { + offset -= sizeof(V128); + + RegV128 value = popV128(); + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushV128(value); + + MemoryAccessDesc access(Scalar::Simd128, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::V128); + + omitBoundsCheck = true; + } +#endif + + freeI32(dest); + freeI32(src); +} + +void BaseCompiler::memFillInlineM32() { + MOZ_ASSERT(MaxInlineMemoryFillLength != 0); + + int32_t signedLength; + int32_t signedValue; + MOZ_ALWAYS_TRUE(popConst(&signedLength)); + MOZ_ALWAYS_TRUE(popConst(&signedValue)); + uint32_t length = uint32_t(signedLength); + uint32_t value = uint32_t(signedValue); + MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryFillLength); + + RegI32 dest = popI32(); + + // Compute the number of copies of each width we will need to do + size_t remainder = length; +#ifdef ENABLE_WASM_SIMD + size_t numCopies16 = 0; + if (MacroAssembler::SupportsFastUnalignedFPAccesses()) { + numCopies16 = remainder / sizeof(V128); + remainder %= sizeof(V128); + } +#endif +#ifdef JS_64BIT + size_t numCopies8 = remainder / sizeof(uint64_t); + remainder %= sizeof(uint64_t); +#endif + size_t numCopies4 = remainder / sizeof(uint32_t); + remainder %= sizeof(uint32_t); + size_t numCopies2 = remainder / sizeof(uint16_t); + remainder %= sizeof(uint16_t); + size_t numCopies1 = remainder; + + MOZ_ASSERT(numCopies2 <= 1 && numCopies1 <= 1); + + // Generate splatted definitions for wider fills as needed +#ifdef ENABLE_WASM_SIMD + V128 val16(value); +#endif +#ifdef JS_64BIT + uint64_t val8 = SplatByteToUInt(value, 8); +#endif + uint32_t val4 = SplatByteToUInt(value, 4); + uint32_t val2 = SplatByteToUInt(value, 2); + uint32_t val1 = value; + + // Store the fill value to the destination from high to low. We will trap + // without writing anything on the first store if any dest byte is + // out-of-bounds. + size_t offset = length; + bool omitBoundsCheck = false; + + if (numCopies1) { + offset -= sizeof(uint8_t); + + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI32(val1); + + MemoryAccessDesc access(Scalar::Uint8, 1, offset, bytecodeOffset()); + AccessCheck check; + storeCommon(&access, check, ValType::I32); + + omitBoundsCheck = true; + } + + if (numCopies2) { + offset -= sizeof(uint16_t); + + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI32(val2); + + MemoryAccessDesc access(Scalar::Uint16, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::I32); + + omitBoundsCheck = true; + } + + for (uint32_t i = 0; i < numCopies4; i++) { + offset -= sizeof(uint32_t); + + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI32(val4); + + MemoryAccessDesc access(Scalar::Uint32, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::I32); + + omitBoundsCheck = true; + } + +#ifdef JS_64BIT + for (uint32_t i = 0; i < numCopies8; i++) { + offset -= sizeof(uint64_t); + + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushI64(val8); + + MemoryAccessDesc access(Scalar::Int64, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::I64); + + omitBoundsCheck = true; + } +#endif + +#ifdef ENABLE_WASM_SIMD + for (uint32_t i = 0; i < numCopies16; i++) { + offset -= sizeof(V128); + + RegI32 temp = needI32(); + moveI32(dest, temp); + pushI32(temp); + pushV128(val16); + + MemoryAccessDesc access(Scalar::Simd128, 1, offset, bytecodeOffset()); + AccessCheck check; + check.omitBoundsCheck = omitBoundsCheck; + storeCommon(&access, check, ValType::V128); + + omitBoundsCheck = true; + } +#endif + + freeI32(dest); +} + +////////////////////////////////////////////////////////////////////////////// +// +// SIMD and Relaxed SIMD. + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::loadSplat(MemoryAccessDesc* access) { + // We can implement loadSplat mostly as load + splat because the push of the + // result onto the value stack in loadCommon normally will not generate any + // code, it will leave the value in a register which we will consume. + + // We use uint types when we can on the general assumption that unsigned loads + // might be smaller/faster on some platforms, because no sign extension needs + // to be done after the sub-register load. + RegV128 rd = needV128(); + switch (access->type()) { + case Scalar::Uint8: { + loadCommon(access, AccessCheck(), ValType::I32); + RegI32 rs = popI32(); + masm.splatX16(rs, rd); + free(rs); + break; + } + case Scalar::Uint16: { + loadCommon(access, AccessCheck(), ValType::I32); + RegI32 rs = popI32(); + masm.splatX8(rs, rd); + free(rs); + break; + } + case Scalar::Uint32: { + loadCommon(access, AccessCheck(), ValType::I32); + RegI32 rs = popI32(); + masm.splatX4(rs, rd); + free(rs); + break; + } + case Scalar::Int64: { + loadCommon(access, AccessCheck(), ValType::I64); + RegI64 rs = popI64(); + masm.splatX2(rs, rd); + free(rs); + break; + } + default: + MOZ_CRASH(); + } + pushV128(rd); +} + +void BaseCompiler::loadZero(MemoryAccessDesc* access) { + access->setZeroExtendSimd128Load(); + loadCommon(access, AccessCheck(), ValType::V128); +} + +void BaseCompiler::loadExtend(MemoryAccessDesc* access, Scalar::Type viewType) { + loadCommon(access, AccessCheck(), ValType::I64); + + RegI64 rs = popI64(); + RegV128 rd = needV128(); + masm.moveGPR64ToDouble(rs, rd); + switch (viewType) { + case Scalar::Int8: + masm.widenLowInt8x16(rd, rd); + break; + case Scalar::Uint8: + masm.unsignedWidenLowInt8x16(rd, rd); + break; + case Scalar::Int16: + masm.widenLowInt16x8(rd, rd); + break; + case Scalar::Uint16: + masm.unsignedWidenLowInt16x8(rd, rd); + break; + case Scalar::Int32: + masm.widenLowInt32x4(rd, rd); + break; + case Scalar::Uint32: + masm.unsignedWidenLowInt32x4(rd, rd); + break; + default: + MOZ_CRASH(); + } + freeI64(rs); + pushV128(rd); +} + +void BaseCompiler::loadLane(MemoryAccessDesc* access, uint32_t laneIndex) { + ValType type = access->type() == Scalar::Int64 ? ValType::I64 : ValType::I32; + + RegV128 rsd = popV128(); + loadCommon(access, AccessCheck(), type); + + if (type == ValType::I32) { + RegI32 rs = popI32(); + switch (access->type()) { + case Scalar::Uint8: + masm.replaceLaneInt8x16(laneIndex, rs, rsd); + break; + case Scalar::Uint16: + masm.replaceLaneInt16x8(laneIndex, rs, rsd); + break; + case Scalar::Int32: + masm.replaceLaneInt32x4(laneIndex, rs, rsd); + break; + default: + MOZ_CRASH("unsupported access type"); + } + freeI32(rs); + } else { + MOZ_ASSERT(type == ValType::I64); + RegI64 rs = popI64(); + masm.replaceLaneInt64x2(laneIndex, rs, rsd); + freeI64(rs); + } + + pushV128(rsd); +} + +void BaseCompiler::storeLane(MemoryAccessDesc* access, uint32_t laneIndex) { + ValType type = access->type() == Scalar::Int64 ? ValType::I64 : ValType::I32; + + RegV128 rs = popV128(); + if (type == ValType::I32) { + RegI32 tmp = needI32(); + switch (access->type()) { + case Scalar::Uint8: + masm.extractLaneInt8x16(laneIndex, rs, tmp); + break; + case Scalar::Uint16: + masm.extractLaneInt16x8(laneIndex, rs, tmp); + break; + case Scalar::Int32: + masm.extractLaneInt32x4(laneIndex, rs, tmp); + break; + default: + MOZ_CRASH("unsupported laneSize"); + } + pushI32(tmp); + } else { + MOZ_ASSERT(type == ValType::I64); + RegI64 tmp = needI64(); + masm.extractLaneInt64x2(laneIndex, rs, tmp); + pushI64(tmp); + } + freeV128(rs); + + storeCommon(access, AccessCheck(), type); +} +#endif // ENABLE_WASM_SIMD + +} // namespace wasm +} // namespace js diff --git a/js/src/wasm/WasmBCRegDefs-inl.h b/js/src/wasm/WasmBCRegDefs-inl.h new file mode 100644 index 0000000000..1531152f03 --- /dev/null +++ b/js/src/wasm/WasmBCRegDefs-inl.h @@ -0,0 +1,180 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: inline BaseCompiler +// methods that don't fit in any other group in particular. + +#ifndef wasm_wasm_baseline_reg_defs_inl_h +#define wasm_wasm_baseline_reg_defs_inl_h + +namespace js { +namespace wasm { +// TODO / OPTIMIZE (Bug 1316802): Do not sync everything on allocation +// failure, only as much as we need. + +RegI32 BaseRegAlloc::needI32() { + if (!hasGPR()) { + bc->sync(); + } + return RegI32(allocGPR()); +} + +void BaseRegAlloc::needI32(RegI32 specific) { + if (!isAvailableI32(specific)) { + bc->sync(); + } + allocGPR(specific); +} + +RegI64 BaseRegAlloc::needI64() { + if (!hasGPR64()) { + bc->sync(); + } + return RegI64(allocInt64()); +} + +void BaseRegAlloc::needI64(RegI64 specific) { + if (!isAvailableI64(specific)) { + bc->sync(); + } + allocInt64(specific); +} + +RegRef BaseRegAlloc::needRef() { + if (!hasGPR()) { + bc->sync(); + } + return RegRef(allocGPR()); +} + +void BaseRegAlloc::needRef(RegRef specific) { + if (!isAvailableRef(specific)) { + bc->sync(); + } + allocGPR(specific); +} + +RegPtr BaseRegAlloc::needPtr() { + if (!hasGPR()) { + bc->sync(); + } + return RegPtr(allocGPR()); +} + +void BaseRegAlloc::needPtr(RegPtr specific) { + if (!isAvailablePtr(specific)) { + bc->sync(); + } + allocGPR(specific); +} + +RegPtr BaseRegAlloc::needTempPtr(RegPtr fallback, bool* saved) { + if (hasGPR()) { + *saved = false; + return RegPtr(allocGPR()); + } + *saved = true; + bc->saveTempPtr(fallback); + MOZ_ASSERT(isAvailablePtr(fallback)); + allocGPR(fallback); + return RegPtr(fallback); +} + +RegF32 BaseRegAlloc::needF32() { + if (!hasFPU()) { + bc->sync(); + } + return RegF32(allocFPU()); +} + +void BaseRegAlloc::needF32(RegF32 specific) { + if (!isAvailableF32(specific)) { + bc->sync(); + } + allocFPU(specific); +} + +RegF64 BaseRegAlloc::needF64() { + if (!hasFPU()) { + bc->sync(); + } + return RegF64(allocFPU()); +} + +void BaseRegAlloc::needF64(RegF64 specific) { + if (!isAvailableF64(specific)) { + bc->sync(); + } + allocFPU(specific); +} + +#ifdef ENABLE_WASM_SIMD +RegV128 BaseRegAlloc::needV128() { + if (!hasFPU()) { + bc->sync(); + } + return RegV128(allocFPU()); +} + +void BaseRegAlloc::needV128(RegV128 specific) { + if (!isAvailableV128(specific)) { + bc->sync(); + } + allocFPU(specific); +} +#endif + +void BaseRegAlloc::freeI32(RegI32 r) { freeGPR(r); } + +void BaseRegAlloc::freeI64(RegI64 r) { freeInt64(r); } + +void BaseRegAlloc::freeRef(RegRef r) { freeGPR(r); } + +void BaseRegAlloc::freePtr(RegPtr r) { freeGPR(r); } + +void BaseRegAlloc::freeF64(RegF64 r) { freeFPU(r); } + +void BaseRegAlloc::freeF32(RegF32 r) { freeFPU(r); } + +#ifdef ENABLE_WASM_SIMD +void BaseRegAlloc::freeV128(RegV128 r) { freeFPU(r); } +#endif + +void BaseRegAlloc::freeTempPtr(RegPtr r, bool saved) { + freePtr(r); + if (saved) { + bc->restoreTempPtr(r); + MOZ_ASSERT(!isAvailablePtr(r)); + } +} + +#ifdef JS_CODEGEN_ARM +RegI64 BaseRegAlloc::needI64Pair() { + if (!hasGPRPair()) { + bc->sync(); + } + Register low, high; + allocGPRPair(&low, &high); + return RegI64(Register64(high, low)); +} +#endif + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_reg_defs_inl_h diff --git a/js/src/wasm/WasmBCRegDefs.h b/js/src/wasm/WasmBCRegDefs.h new file mode 100644 index 0000000000..621ee7c6a9 --- /dev/null +++ b/js/src/wasm/WasmBCRegDefs.h @@ -0,0 +1,852 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: definitions of +// registers and the register allocator. + +#ifndef wasm_wasm_baseline_regdefs_h +#define wasm_wasm_baseline_regdefs_h + +#include "wasm/WasmBCDefs.h" + +namespace js { +namespace wasm { + +struct BaseCompiler; + +using namespace js::jit; + +////////////////////////////////////////////////////////////////////////////// +// +// Scratch register configuration. + +#if defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) +# define RABALDR_SCRATCH_I32 +# define RABALDR_SCRATCH_F32 +# define RABALDR_SCRATCH_F64 + +static constexpr Register RabaldrScratchI32 = Register::Invalid(); +static constexpr FloatRegister RabaldrScratchF32 = InvalidFloatReg; +static constexpr FloatRegister RabaldrScratchF64 = InvalidFloatReg; +#endif + +#ifdef JS_CODEGEN_ARM64 +# define RABALDR_SCRATCH_I32 +# define RABALDR_SCRATCH_F32 +# define RABALDR_SCRATCH_F64 +# define RABALDR_SCRATCH_V128 +# define RABALDR_SCRATCH_F32_ALIASES_F64 + +static constexpr Register RabaldrScratchI32{Registers::x15}; + +// Note, the float scratch regs cannot be registers that are used for parameter +// passing in any ABI we use. Argregs tend to be low-numbered; register 30 +// should be safe. + +static constexpr FloatRegister RabaldrScratchF32{FloatRegisters::s30, + FloatRegisters::Single}; +static constexpr FloatRegister RabaldrScratchF64{FloatRegisters::d30, + FloatRegisters::Double}; +# ifdef ENABLE_WASM_SIMD +static constexpr FloatRegister RabaldrScratchV128{FloatRegisters::d30, + FloatRegisters::Simd128}; +# endif + +static_assert(RabaldrScratchF32 != ScratchFloat32Reg_, "Too busy"); +static_assert(RabaldrScratchF64 != ScratchDoubleReg_, "Too busy"); +# ifdef ENABLE_WASM_SIMD +static_assert(RabaldrScratchV128 != ScratchSimd128Reg, "Too busy"); +# endif +#endif + +#ifdef JS_CODEGEN_X86 +// The selection of EBX here steps gingerly around: the need for EDX +// to be allocatable for multiply/divide; ECX to be allocatable for +// shift/rotate; EAX (= ReturnReg) to be allocatable as the result +// register; EBX not being one of the WasmTableCall registers; and +// needing a temp register for load/store that has a single-byte +// persona. +// +// The compiler assumes that RabaldrScratchI32 has a single-byte +// persona. Code for 8-byte atomic operations assumes that +// RabaldrScratchI32 is in fact ebx. + +# define RABALDR_SCRATCH_I32 +static constexpr Register RabaldrScratchI32 = ebx; +#endif + +#ifdef JS_CODEGEN_ARM +// We use our own scratch register, because the macro assembler uses +// the regular scratch register(s) pretty liberally. We could +// work around that in several cases but the mess does not seem +// worth it yet. CallTempReg2 seems safe. + +# define RABALDR_SCRATCH_I32 +static constexpr Register RabaldrScratchI32 = CallTempReg2; +#endif + +#ifdef JS_CODEGEN_MIPS64 +# define RABALDR_SCRATCH_I32 +static constexpr Register RabaldrScratchI32 = CallTempReg2; +#endif + +#ifdef JS_CODEGEN_LOONG64 +// We use our own scratch register, because the macro assembler uses +// the regular scratch register(s) pretty liberally. We could +// work around that in several cases but the mess does not seem +// worth it yet. CallTempReg2 seems safe. + +# define RABALDR_SCRATCH_I32 +static constexpr Register RabaldrScratchI32 = CallTempReg2; +#endif + +#ifdef RABALDR_SCRATCH_F32_ALIASES_F64 +# if !defined(RABALDR_SCRATCH_F32) || !defined(RABALDR_SCRATCH_F64) +# error "Bad configuration" +# endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// ... + +template +struct RegTypeOf { +#ifdef ENABLE_WASM_SIMD + static_assert(t == MIRType::Float32 || t == MIRType::Double || + t == MIRType::Simd128, + "Float mask type"); +#else + static_assert(t == MIRType::Float32 || t == MIRType::Double, + "Float mask type"); +#endif +}; + +template <> +struct RegTypeOf { + static constexpr RegTypeName value = RegTypeName::Float32; +}; +template <> +struct RegTypeOf { + static constexpr RegTypeName value = RegTypeName::Float64; +}; +#ifdef ENABLE_WASM_SIMD +template <> +struct RegTypeOf { + static constexpr RegTypeName value = RegTypeName::Vector128; +}; +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Strongly typed register wrappers. + +// The strongly typed register wrappers are especially useful to distinguish +// float registers from double registers, but they also clearly distinguish +// 32-bit registers from 64-bit register pairs on 32-bit systems. + +struct RegI32 : public Register { + RegI32() : Register(Register::Invalid()) {} + explicit RegI32(Register reg) : Register(reg) { + MOZ_ASSERT(reg != Invalid()); + } + bool isInvalid() const { return *this == Invalid(); } + bool isValid() const { return !isInvalid(); } + static RegI32 Invalid() { return RegI32(); } +}; + +struct RegI64 : public Register64 { + RegI64() : Register64(Register64::Invalid()) {} + explicit RegI64(Register64 reg) : Register64(reg) { + MOZ_ASSERT(reg != Invalid()); + } + bool isInvalid() const { return *this == Invalid(); } + bool isValid() const { return !isInvalid(); } + static RegI64 Invalid() { return RegI64(); } +}; + +// RegRef is for GC-pointers, for non GC-pointers use RegPtr +struct RegRef : public Register { + RegRef() : Register(Register::Invalid()) {} + explicit RegRef(Register reg) : Register(reg) { + MOZ_ASSERT(reg != Invalid()); + } + bool isInvalid() const { return *this == Invalid(); } + bool isValid() const { return !isInvalid(); } + static RegRef Invalid() { return RegRef(); } +}; + +// RegPtr is for non GC-pointers, for GC-pointers use RegRef +struct RegPtr : public Register { + RegPtr() : Register(Register::Invalid()) {} + explicit RegPtr(Register reg) : Register(reg) { + MOZ_ASSERT(reg != Invalid()); + } + bool isInvalid() const { return *this == Invalid(); } + bool isValid() const { return !isInvalid(); } + static RegPtr Invalid() { return RegPtr(); } +}; + +struct RegF32 : public FloatRegister { + RegF32() : FloatRegister() {} + explicit RegF32(FloatRegister reg) : FloatRegister(reg) { + MOZ_ASSERT(isSingle()); + } + bool isValid() const { return !isInvalid(); } + static RegF32 Invalid() { return RegF32(); } +}; + +struct RegF64 : public FloatRegister { + RegF64() : FloatRegister() {} + explicit RegF64(FloatRegister reg) : FloatRegister(reg) { + MOZ_ASSERT(isDouble()); + } + bool isValid() const { return !isInvalid(); } + static RegF64 Invalid() { return RegF64(); } +}; + +#ifdef ENABLE_WASM_SIMD +struct RegV128 : public FloatRegister { + RegV128() : FloatRegister() {} + explicit RegV128(FloatRegister reg) : FloatRegister(reg) { + MOZ_ASSERT(isSimd128()); + } + bool isValid() const { return !isInvalid(); } + static RegV128 Invalid() { return RegV128(); } +}; +#endif + +struct AnyReg { + union { + RegI32 i32_; + RegI64 i64_; + RegRef ref_; + RegF32 f32_; + RegF64 f64_; +#ifdef ENABLE_WASM_SIMD + RegV128 v128_; +#endif + }; + + enum { + I32, + I64, + REF, + F32, + F64, +#ifdef ENABLE_WASM_SIMD + V128 +#endif + } tag; + + explicit AnyReg(RegI32 r) { + tag = I32; + i32_ = r; + } + explicit AnyReg(RegI64 r) { + tag = I64; + i64_ = r; + } + explicit AnyReg(RegF32 r) { + tag = F32; + f32_ = r; + } + explicit AnyReg(RegF64 r) { + tag = F64; + f64_ = r; + } +#ifdef ENABLE_WASM_SIMD + explicit AnyReg(RegV128 r) { + tag = V128; + v128_ = r; + } +#endif + explicit AnyReg(RegRef r) { + tag = REF; + ref_ = r; + } + + RegI32 i32() const { + MOZ_ASSERT(tag == I32); + return i32_; + } + RegI64 i64() const { + MOZ_ASSERT(tag == I64); + return i64_; + } + RegF32 f32() const { + MOZ_ASSERT(tag == F32); + return f32_; + } + RegF64 f64() const { + MOZ_ASSERT(tag == F64); + return f64_; + } +#ifdef ENABLE_WASM_SIMD + RegV128 v128() const { + MOZ_ASSERT(tag == V128); + return v128_; + } +#endif + RegRef ref() const { + MOZ_ASSERT(tag == REF); + return ref_; + } + + AnyRegister any() const { + switch (tag) { + case F32: + return AnyRegister(f32_); + case F64: + return AnyRegister(f64_); +#ifdef ENABLE_WASM_SIMD + case V128: + return AnyRegister(v128_); +#endif + case I32: + return AnyRegister(i32_); + case I64: +#ifdef JS_PUNBOX64 + return AnyRegister(i64_.reg); +#else + // The compiler is written so that this is never needed: any() is + // called on arbitrary registers for asm.js but asm.js does not have + // 64-bit ints. For wasm, any() is called on arbitrary registers + // only on 64-bit platforms. + MOZ_CRASH("AnyReg::any() on 32-bit platform"); +#endif + case REF: + MOZ_CRASH("AnyReg::any() not implemented for ref types"); + default: + MOZ_CRASH(); + } + // Work around GCC 5 analysis/warning bug. + MOZ_CRASH("AnyReg::any(): impossible case"); + } +}; + +////////////////////////////////////////////////////////////////////////////// +// +// Platform-specific registers. +// +// All platforms must define struct SpecificRegs. All 32-bit platforms must +// have an abiReturnRegI64 member in that struct. + +#if defined(JS_CODEGEN_X64) +struct SpecificRegs { + RegI32 eax, ecx, edx, edi, esi; + RegI64 rax, rcx, rdx; + + SpecificRegs() + : eax(RegI32(js::jit::eax)), + ecx(RegI32(js::jit::ecx)), + edx(RegI32(js::jit::edx)), + edi(RegI32(js::jit::edi)), + esi(RegI32(js::jit::esi)), + rax(RegI64(Register64(js::jit::rax))), + rcx(RegI64(Register64(js::jit::rcx))), + rdx(RegI64(Register64(js::jit::rdx))) {} +}; +#elif defined(JS_CODEGEN_X86) +struct SpecificRegs { + RegI32 eax, ecx, edx, edi, esi; + RegI64 ecx_ebx, edx_eax, abiReturnRegI64; + + SpecificRegs() + : eax(RegI32(js::jit::eax)), + ecx(RegI32(js::jit::ecx)), + edx(RegI32(js::jit::edx)), + edi(RegI32(js::jit::edi)), + esi(RegI32(js::jit::esi)), + ecx_ebx(RegI64(Register64(js::jit::ecx, js::jit::ebx))), + edx_eax(RegI64(Register64(js::jit::edx, js::jit::eax))), + abiReturnRegI64(edx_eax) {} +}; +#elif defined(JS_CODEGEN_ARM) +struct SpecificRegs { + RegI64 abiReturnRegI64; + + SpecificRegs() : abiReturnRegI64(ReturnReg64) {} +}; +#elif defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) +struct SpecificRegs { + // Required by gcc. + SpecificRegs() {} +}; +#else +struct SpecificRegs { +# ifndef JS_64BIT + RegI64 abiReturnRegI64; +# endif + + SpecificRegs() { MOZ_CRASH("BaseCompiler porting interface: SpecificRegs"); } +}; +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Register allocator. + +class BaseRegAlloc { + // Notes on float register allocation. + // + // The general rule in SpiderMonkey is that float registers can alias double + // registers, but there are predicates to handle exceptions to that rule: + // hasUnaliasedDouble() and hasMultiAlias(). The way aliasing actually + // works is platform dependent and exposed through the aliased(n, &r) + // predicate, etc. + // + // - hasUnaliasedDouble(): on ARM VFPv3-D32 there are double registers that + // cannot be treated as float. + // - hasMultiAlias(): on ARM and MIPS a double register aliases two float + // registers. + // + // On some platforms (x86, x64, ARM64) but not all (ARM) + // ScratchFloat32Register is the same as ScratchDoubleRegister. + // + // It's a basic invariant of the AllocatableRegisterSet that it deals + // properly with aliasing of registers: if s0 or s1 are allocated then d0 is + // not allocatable; if s0 and s1 are freed individually then d0 becomes + // allocatable. + + BaseCompiler* bc; + AllocatableGeneralRegisterSet availGPR; + AllocatableFloatRegisterSet availFPU; +#ifdef DEBUG + // The registers available after removing ScratchReg, HeapReg, etc. + AllocatableGeneralRegisterSet allGPR; + AllocatableFloatRegisterSet allFPU; + uint32_t scratchTaken; +#endif +#ifdef JS_CODEGEN_X86 + AllocatableGeneralRegisterSet singleByteRegs; +#endif + + bool hasGPR() { return !availGPR.empty(); } + + bool hasGPR64() { +#ifdef JS_PUNBOX64 + return !availGPR.empty(); +#else + if (availGPR.empty()) { + return false; + } + Register r = allocGPR(); + bool available = !availGPR.empty(); + freeGPR(r); + return available; +#endif + } + + template + bool hasFPU() { + return availFPU.hasAny::value>(); + } + + bool isAvailableGPR(Register r) { return availGPR.has(r); } + + bool isAvailableFPU(FloatRegister r) { return availFPU.has(r); } + + void allocGPR(Register r) { + MOZ_ASSERT(isAvailableGPR(r)); + availGPR.take(r); + } + + Register allocGPR() { + MOZ_ASSERT(hasGPR()); + return availGPR.takeAny(); + } + + void allocInt64(Register64 r) { +#ifdef JS_PUNBOX64 + allocGPR(r.reg); +#else + allocGPR(r.low); + allocGPR(r.high); +#endif + } + + Register64 allocInt64() { + MOZ_ASSERT(hasGPR64()); +#ifdef JS_PUNBOX64 + return Register64(availGPR.takeAny()); +#else + Register high = availGPR.takeAny(); + Register low = availGPR.takeAny(); + return Register64(high, low); +#endif + } + +#ifdef JS_CODEGEN_ARM + // r12 is normally the ScratchRegister and r13 is always the stack pointer, + // so the highest possible pair has r10 as the even-numbered register. + + static constexpr uint32_t PAIR_LIMIT = 10; + + bool hasGPRPair() { + for (uint32_t i = 0; i <= PAIR_LIMIT; i += 2) { + if (isAvailableGPR(Register::FromCode(i)) && + isAvailableGPR(Register::FromCode(i + 1))) { + return true; + } + } + return false; + } + + void allocGPRPair(Register* low, Register* high) { + MOZ_ASSERT(hasGPRPair()); + for (uint32_t i = 0; i <= PAIR_LIMIT; i += 2) { + if (isAvailableGPR(Register::FromCode(i)) && + isAvailableGPR(Register::FromCode(i + 1))) { + *low = Register::FromCode(i); + *high = Register::FromCode(i + 1); + allocGPR(*low); + allocGPR(*high); + return; + } + } + MOZ_CRASH("No pair"); + } +#endif + + void allocFPU(FloatRegister r) { + MOZ_ASSERT(isAvailableFPU(r)); + availFPU.take(r); + } + + template + FloatRegister allocFPU() { + return availFPU.takeAny::value>(); + } + + void freeGPR(Register r) { availGPR.add(r); } + + void freeInt64(Register64 r) { +#ifdef JS_PUNBOX64 + freeGPR(r.reg); +#else + freeGPR(r.low); + freeGPR(r.high); +#endif + } + + void freeFPU(FloatRegister r) { availFPU.add(r); } + + public: + explicit BaseRegAlloc() + : bc(nullptr), + availGPR(GeneralRegisterSet::All()), + availFPU(FloatRegisterSet::All()) +#ifdef DEBUG + , + scratchTaken(0) +#endif +#ifdef JS_CODEGEN_X86 + , + singleByteRegs(GeneralRegisterSet(Registers::SingleByteRegs)) +#endif + { + RegisterAllocator::takeWasmRegisters(availGPR); + +#ifdef RABALDR_PIN_INSTANCE + // If the InstanceReg is pinned then it is never available for + // allocation. + availGPR.take(InstanceReg); +#endif + + // Allocate any private scratch registers. +#if defined(RABALDR_SCRATCH_I32) + if (RabaldrScratchI32 != RegI32::Invalid()) { + availGPR.take(RabaldrScratchI32); + } +#endif + +#ifdef RABALDR_SCRATCH_F32_ALIASES_F64 + static_assert(RabaldrScratchF32 != InvalidFloatReg, "Float reg definition"); + static_assert(RabaldrScratchF64 != InvalidFloatReg, "Float reg definition"); +#endif + +#if defined(RABALDR_SCRATCH_F32) && !defined(RABALDR_SCRATCH_F32_ALIASES_F64) + if (RabaldrScratchF32 != RegF32::Invalid()) { + availFPU.take(RabaldrScratchF32); + } +#endif + +#if defined(RABALDR_SCRATCH_F64) +# ifdef RABALDR_SCRATCH_F32_ALIASES_F64 + MOZ_ASSERT(availFPU.has(RabaldrScratchF32)); +# endif + if (RabaldrScratchF64 != RegF64::Invalid()) { + availFPU.take(RabaldrScratchF64); + } +# ifdef RABALDR_SCRATCH_F32_ALIASES_F64 + MOZ_ASSERT(!availFPU.has(RabaldrScratchF32)); +# endif +#endif + +#ifdef DEBUG + allGPR = availGPR; + allFPU = availFPU; +#endif + } + + void init(BaseCompiler* bc) { this->bc = bc; } + + enum class ScratchKind { I32 = 1, F32 = 2, F64 = 4, V128 = 8 }; + +#ifdef DEBUG + bool isScratchRegisterTaken(ScratchKind s) const { + return (scratchTaken & uint32_t(s)) != 0; + } + + void setScratchRegisterTaken(ScratchKind s, bool state) { + if (state) { + scratchTaken |= uint32_t(s); + } else { + scratchTaken &= ~uint32_t(s); + } + } +#endif + +#ifdef JS_CODEGEN_X86 + bool isSingleByteI32(Register r) { return singleByteRegs.has(r); } +#endif + + bool isAvailableI32(RegI32 r) { return isAvailableGPR(r); } + + bool isAvailableI64(RegI64 r) { +#ifdef JS_PUNBOX64 + return isAvailableGPR(r.reg); +#else + return isAvailableGPR(r.low) && isAvailableGPR(r.high); +#endif + } + + bool isAvailableRef(RegRef r) { return isAvailableGPR(r); } + + bool isAvailablePtr(RegPtr r) { return isAvailableGPR(r); } + + bool isAvailableF32(RegF32 r) { return isAvailableFPU(r); } + + bool isAvailableF64(RegF64 r) { return isAvailableFPU(r); } + +#ifdef ENABLE_WASM_SIMD + bool isAvailableV128(RegV128 r) { return isAvailableFPU(r); } +#endif + + [[nodiscard]] inline RegI32 needI32(); + inline void needI32(RegI32 specific); + + [[nodiscard]] inline RegI64 needI64(); + inline void needI64(RegI64 specific); + + [[nodiscard]] inline RegRef needRef(); + inline void needRef(RegRef specific); + + [[nodiscard]] inline RegPtr needPtr(); + inline void needPtr(RegPtr specific); + + [[nodiscard]] inline RegF32 needF32(); + inline void needF32(RegF32 specific); + + [[nodiscard]] inline RegF64 needF64(); + inline void needF64(RegF64 specific); + +#ifdef ENABLE_WASM_SIMD + [[nodiscard]] inline RegV128 needV128(); + inline void needV128(RegV128 specific); +#endif + + inline void freeI32(RegI32 r); + inline void freeI64(RegI64 r); + inline void freeRef(RegRef r); + inline void freePtr(RegPtr r); + inline void freeF64(RegF64 r); + inline void freeF32(RegF32 r); +#ifdef ENABLE_WASM_SIMD + inline void freeV128(RegV128 r); +#endif + + // Use when you need a register for a short time but explicitly want to avoid + // a full sync(). + [[nodiscard]] inline RegPtr needTempPtr(RegPtr fallback, bool* saved); + inline void freeTempPtr(RegPtr r, bool saved); + +#ifdef JS_CODEGEN_ARM + [[nodiscard]] inline RegI64 needI64Pair(); +#endif + +#ifdef DEBUG + friend class LeakCheck; + + class MOZ_RAII LeakCheck { + private: + const BaseRegAlloc& ra; + AllocatableGeneralRegisterSet knownGPR_; + AllocatableFloatRegisterSet knownFPU_; + + public: + explicit LeakCheck(const BaseRegAlloc& ra) : ra(ra) { + knownGPR_ = ra.availGPR; + knownFPU_ = ra.availFPU; + } + + ~LeakCheck() { + MOZ_ASSERT(knownGPR_.bits() == ra.allGPR.bits()); + MOZ_ASSERT(knownFPU_.bits() == ra.allFPU.bits()); + } + + void addKnownI32(RegI32 r) { knownGPR_.add(r); } + + void addKnownI64(RegI64 r) { +# ifdef JS_PUNBOX64 + knownGPR_.add(r.reg); +# else + knownGPR_.add(r.high); + knownGPR_.add(r.low); +# endif + } + + void addKnownF32(RegF32 r) { knownFPU_.add(r); } + + void addKnownF64(RegF64 r) { knownFPU_.add(r); } + +# ifdef ENABLE_WASM_SIMD + void addKnownV128(RegV128 r) { knownFPU_.add(r); } +# endif + + void addKnownRef(RegRef r) { knownGPR_.add(r); } + }; +#endif +}; + +// Scratch register abstractions. +// +// We define our own scratch registers when the platform doesn't provide what we +// need. A notable use case is that we will need a private scratch register +// when the platform masm uses its scratch register very frequently (eg, ARM). + +class BaseScratchRegister { +#ifdef DEBUG + BaseRegAlloc& ra; + BaseRegAlloc::ScratchKind kind_; + + public: + explicit BaseScratchRegister(BaseRegAlloc& ra, BaseRegAlloc::ScratchKind kind) + : ra(ra), kind_(kind) { + MOZ_ASSERT(!ra.isScratchRegisterTaken(kind_)); + ra.setScratchRegisterTaken(kind_, true); + } + ~BaseScratchRegister() { + MOZ_ASSERT(ra.isScratchRegisterTaken(kind_)); + ra.setScratchRegisterTaken(kind_, false); + } +#else + public: + explicit BaseScratchRegister(BaseRegAlloc& ra, + BaseRegAlloc::ScratchKind kind) {} +#endif +}; + +#ifdef ENABLE_WASM_SIMD +# ifdef RABALDR_SCRATCH_V128 +class ScratchV128 : public BaseScratchRegister { + public: + explicit ScratchV128(BaseRegAlloc& ra) + : BaseScratchRegister(ra, BaseRegAlloc::ScratchKind::V128) {} + operator RegV128() const { return RegV128(RabaldrScratchV128); } +}; +# else +class ScratchV128 : public ScratchSimd128Scope { + public: + explicit ScratchV128(MacroAssembler& m) : ScratchSimd128Scope(m) {} + operator RegV128() const { return RegV128(FloatRegister(*this)); } +}; +# endif +#endif + +#ifdef RABALDR_SCRATCH_F64 +class ScratchF64 : public BaseScratchRegister { + public: + explicit ScratchF64(BaseRegAlloc& ra) + : BaseScratchRegister(ra, BaseRegAlloc::ScratchKind::F64) {} + operator RegF64() const { return RegF64(RabaldrScratchF64); } +}; +#else +class ScratchF64 : public ScratchDoubleScope { + public: + explicit ScratchF64(MacroAssembler& m) : ScratchDoubleScope(m) {} + operator RegF64() const { return RegF64(FloatRegister(*this)); } +}; +#endif + +#ifdef RABALDR_SCRATCH_F32 +class ScratchF32 : public BaseScratchRegister { + public: + explicit ScratchF32(BaseRegAlloc& ra) + : BaseScratchRegister(ra, BaseRegAlloc::ScratchKind::F32) {} + operator RegF32() const { return RegF32(RabaldrScratchF32); } +}; +#else +class ScratchF32 : public ScratchFloat32Scope { + public: + explicit ScratchF32(MacroAssembler& m) : ScratchFloat32Scope(m) {} + operator RegF32() const { return RegF32(FloatRegister(*this)); } +}; +#endif + +#ifdef RABALDR_SCRATCH_I32 +template +class ScratchGPR : public BaseScratchRegister { + public: + explicit ScratchGPR(BaseRegAlloc& ra) + : BaseScratchRegister(ra, BaseRegAlloc::ScratchKind::I32) {} + operator RegType() const { return RegType(RabaldrScratchI32); } +}; +#else +template +class ScratchGPR : public ScratchRegisterScope { + public: + explicit ScratchGPR(MacroAssembler& m) : ScratchRegisterScope(m) {} + operator RegType() const { return RegType(Register(*this)); } +}; +#endif + +using ScratchI32 = ScratchGPR; +using ScratchPtr = ScratchGPR; +using ScratchRef = ScratchGPR; + +#if defined(JS_CODEGEN_X86) +// ScratchEBX is a mnemonic device: For some atomic ops we really need EBX, +// no other register will do. And we would normally have to allocate that +// register using ScratchI32 since normally the scratch register is EBX. +// But the whole point of ScratchI32 is to hide that relationship. By using +// the ScratchEBX alias, we document that at that point we require the +// scratch register to be EBX. +using ScratchEBX = ScratchI32; + +// ScratchI8 is a mnemonic device: For some ops we need a register with a +// byte subregister. +using ScratchI8 = ScratchI32; +#endif + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_regdefs_h diff --git a/js/src/wasm/WasmBCRegMgmt-inl.h b/js/src/wasm/WasmBCRegMgmt-inl.h new file mode 100644 index 0000000000..d448b7405b --- /dev/null +++ b/js/src/wasm/WasmBCRegMgmt-inl.h @@ -0,0 +1,486 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: inline methods in the +// compiler for register management. + +#ifndef wasm_wasm_baseline_reg_mgmt_inl_h +#define wasm_wasm_baseline_reg_mgmt_inl_h + +namespace js { +namespace wasm { + +bool BaseCompiler::isAvailableI32(RegI32 r) { return ra.isAvailableI32(r); } +bool BaseCompiler::isAvailableI64(RegI64 r) { return ra.isAvailableI64(r); } +bool BaseCompiler::isAvailableRef(RegRef r) { return ra.isAvailableRef(r); } +bool BaseCompiler::isAvailablePtr(RegPtr r) { return ra.isAvailablePtr(r); } +bool BaseCompiler::isAvailableF32(RegF32 r) { return ra.isAvailableF32(r); } +bool BaseCompiler::isAvailableF64(RegF64 r) { return ra.isAvailableF64(r); } +#ifdef ENABLE_WASM_SIMD +bool BaseCompiler::isAvailableV128(RegV128 r) { return ra.isAvailableV128(r); } +#endif + +[[nodiscard]] RegI32 BaseCompiler::needI32() { return ra.needI32(); } +[[nodiscard]] RegI64 BaseCompiler::needI64() { return ra.needI64(); } +[[nodiscard]] RegRef BaseCompiler::needRef() { return ra.needRef(); } +[[nodiscard]] RegPtr BaseCompiler::needPtr() { return ra.needPtr(); } +[[nodiscard]] RegF32 BaseCompiler::needF32() { return ra.needF32(); } +[[nodiscard]] RegF64 BaseCompiler::needF64() { return ra.needF64(); } +#ifdef ENABLE_WASM_SIMD +[[nodiscard]] RegV128 BaseCompiler::needV128() { return ra.needV128(); } +#endif + +void BaseCompiler::needI32(RegI32 specific) { ra.needI32(specific); } +void BaseCompiler::needI64(RegI64 specific) { ra.needI64(specific); } +void BaseCompiler::needRef(RegRef specific) { ra.needRef(specific); } +void BaseCompiler::needPtr(RegPtr specific) { ra.needPtr(specific); } +void BaseCompiler::needF32(RegF32 specific) { ra.needF32(specific); } +void BaseCompiler::needF64(RegF64 specific) { ra.needF64(specific); } +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::needV128(RegV128 specific) { ra.needV128(specific); } +#endif + +#if defined(JS_CODEGEN_ARM) +[[nodiscard]] RegI64 BaseCompiler::needI64Pair() { return ra.needI64Pair(); } +#endif + +void BaseCompiler::freeI32(RegI32 r) { ra.freeI32(r); } +void BaseCompiler::freeI64(RegI64 r) { ra.freeI64(r); } +void BaseCompiler::freeRef(RegRef r) { ra.freeRef(r); } +void BaseCompiler::freePtr(RegPtr r) { ra.freePtr(r); } +void BaseCompiler::freeF32(RegF32 r) { ra.freeF32(r); } +void BaseCompiler::freeF64(RegF64 r) { ra.freeF64(r); } +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::freeV128(RegV128 r) { ra.freeV128(r); } +#endif + +void BaseCompiler::freeAny(AnyReg r) { + switch (r.tag) { + case AnyReg::I32: + freeI32(r.i32()); + break; + case AnyReg::I64: + freeI64(r.i64()); + break; + case AnyReg::REF: + freeRef(r.ref()); + break; + case AnyReg::F32: + freeF32(r.f32()); + break; + case AnyReg::F64: + freeF64(r.f64()); + break; +#ifdef ENABLE_WASM_SIMD + case AnyReg::V128: + freeV128(r.v128()); + break; +#endif + default: + MOZ_CRASH(); + } +} + +template <> +inline void BaseCompiler::free(RegI32 r) { + freeI32(r); +} + +template <> +inline void BaseCompiler::free(RegI64 r) { + freeI64(r); +} + +template <> +inline void BaseCompiler::free(RegRef r) { + freeRef(r); +} + +template <> +inline void BaseCompiler::free(RegPtr r) { + freePtr(r); +} + +template <> +inline void BaseCompiler::free(RegF32 r) { + freeF32(r); +} + +template <> +inline void BaseCompiler::free(RegF64 r) { + freeF64(r); +} + +#ifdef ENABLE_WASM_SIMD +template <> +inline void BaseCompiler::free(RegV128 r) { + freeV128(r); +} +#endif + +template <> +inline void BaseCompiler::free(AnyReg r) { + freeAny(r); +} + +void BaseCompiler::freeI64Except(RegI64 r, RegI32 except) { +#ifdef JS_PUNBOX64 + MOZ_ASSERT(r.reg == except); +#else + MOZ_ASSERT(r.high == except || r.low == except); + freeI64(r); + needI32(except); +#endif +} + +void BaseCompiler::maybeFree(RegI32 r) { + if (r.isValid()) { + freeI32(r); + } +} + +void BaseCompiler::maybeFree(RegI64 r) { + if (r.isValid()) { + freeI64(r); + } +} + +void BaseCompiler::maybeFree(RegF32 r) { + if (r.isValid()) { + freeF32(r); + } +} + +void BaseCompiler::maybeFree(RegF64 r) { + if (r.isValid()) { + freeF64(r); + } +} + +void BaseCompiler::maybeFree(RegRef r) { + if (r.isValid()) { + freeRef(r); + } +} + +void BaseCompiler::maybeFree(RegPtr r) { + if (r.isValid()) { + freePtr(r); + } +} + +#ifdef ENABLE_WASM_SIMD128 +void BaseCompiler::maybeFree(RegV128 r) { + if (r.isValid()) { + freeV128(r); + } +} +#endif + +void BaseCompiler::needI32NoSync(RegI32 r) { + MOZ_ASSERT(isAvailableI32(r)); + needI32(r); +} + +// TODO / OPTIMIZE: need2xI32() can be optimized along with needI32() +// to avoid sync(). (Bug 1316802) + +void BaseCompiler::need2xI32(RegI32 r0, RegI32 r1) { + needI32(r0); + needI32(r1); +} + +void BaseCompiler::need2xI64(RegI64 r0, RegI64 r1) { + needI64(r0); + needI64(r1); +} + +RegI32 BaseCompiler::fromI64(RegI64 r) { return RegI32(lowPart(r)); } + +RegI32 BaseCompiler::maybeFromI64(RegI64 r) { + if (!r.isValid()) { + return RegI32::Invalid(); + } + return fromI64(r); +} + +#ifdef JS_PUNBOX64 +RegI64 BaseCompiler::fromI32(RegI32 r) { return RegI64(Register64(r)); } +#endif + +RegI64 BaseCompiler::widenI32(RegI32 r) { + MOZ_ASSERT(!isAvailableI32(r)); +#ifdef JS_PUNBOX64 + return fromI32(r); +#else + RegI32 high = needI32(); + return RegI64(Register64(high, r)); +#endif +} + +RegI32 BaseCompiler::narrowI64(RegI64 r) { +#ifdef JS_PUNBOX64 + return RegI32(r.reg); +#else + freeI32(RegI32(r.high)); + return RegI32(r.low); +#endif +} + +RegI32 BaseCompiler::narrowRef(RegRef r) { return RegI32(r); } + +RegI32 BaseCompiler::lowPart(RegI64 r) { +#ifdef JS_PUNBOX64 + return RegI32(r.reg); +#else + return RegI32(r.low); +#endif +} + +RegI32 BaseCompiler::maybeHighPart(RegI64 r) { +#ifdef JS_PUNBOX64 + return RegI32::Invalid(); +#else + return RegI32(r.high); +#endif +} + +void BaseCompiler::maybeClearHighPart(RegI64 r) { +#if !defined(JS_PUNBOX64) + moveImm32(0, RegI32(r.high)); +#endif +} + +// TODO: We want these to be inlined for sure; do we need an `inline` somewhere? + +template <> +inline RegI32 BaseCompiler::need() { + return needI32(); +} +template <> +inline RegI64 BaseCompiler::need() { + return needI64(); +} +template <> +inline RegF32 BaseCompiler::need() { + return needF32(); +} +template <> +inline RegF64 BaseCompiler::need() { + return needF64(); +} + +template <> +inline RegI32 BaseCompiler::pop() { + return popI32(); +} +template <> +inline RegI64 BaseCompiler::pop() { + return popI64(); +} +template <> +inline RegF32 BaseCompiler::pop() { + return popF32(); +} +template <> +inline RegF64 BaseCompiler::pop() { + return popF64(); +} + +#ifdef ENABLE_WASM_SIMD +template <> +inline RegV128 BaseCompiler::need() { + return needV128(); +} +template <> +inline RegV128 BaseCompiler::pop() { + return popV128(); +} +#endif + +// RegPtr values can't be pushed, hence can't be popped. +template <> +inline RegPtr BaseCompiler::need() { + return needPtr(); +} + +void BaseCompiler::needResultRegisters(ResultType type, ResultRegKind which) { + if (type.empty()) { + return; + } + + for (ABIResultIter iter(type); !iter.done(); iter.next()) { + ABIResult result = iter.cur(); + // Register results are visited first; when we see a stack result we're + // done. + if (!result.inRegister()) { + return; + } + switch (result.type().kind()) { + case ValType::I32: + needI32(RegI32(result.gpr())); + break; + case ValType::I64: + needI64(RegI64(result.gpr64())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + if (which == ResultRegKind::All) { + needV128(RegV128(result.fpr())); + } + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + case ValType::F32: + if (which == ResultRegKind::All) { + needF32(RegF32(result.fpr())); + } + break; + case ValType::F64: + if (which == ResultRegKind::All) { + needF64(RegF64(result.fpr())); + } + break; + case ValType::Ref: + needRef(RegRef(result.gpr())); + break; + } + } +} + +#ifdef JS_64BIT +void BaseCompiler::widenInt32ResultRegisters(ResultType type) { + if (type.empty()) { + return; + } + + for (ABIResultIter iter(type); !iter.done(); iter.next()) { + ABIResult result = iter.cur(); + if (result.inRegister() && result.type().kind() == ValType::I32) { + masm.widenInt32(result.gpr()); + } + } +} +#endif + +void BaseCompiler::freeResultRegisters(ResultType type, ResultRegKind which) { + if (type.empty()) { + return; + } + + for (ABIResultIter iter(type); !iter.done(); iter.next()) { + ABIResult result = iter.cur(); + // Register results are visited first; when we see a stack result we're + // done. + if (!result.inRegister()) { + return; + } + switch (result.type().kind()) { + case ValType::I32: + freeI32(RegI32(result.gpr())); + break; + case ValType::I64: + freeI64(RegI64(result.gpr64())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + if (which == ResultRegKind::All) { + freeV128(RegV128(result.fpr())); + } + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + case ValType::F32: + if (which == ResultRegKind::All) { + freeF32(RegF32(result.fpr())); + } + break; + case ValType::F64: + if (which == ResultRegKind::All) { + freeF64(RegF64(result.fpr())); + } + break; + case ValType::Ref: + freeRef(RegRef(result.gpr())); + break; + } + } +} + +void BaseCompiler::needIntegerResultRegisters(ResultType type) { + needResultRegisters(type, ResultRegKind::OnlyGPRs); +} + +void BaseCompiler::freeIntegerResultRegisters(ResultType type) { + freeResultRegisters(type, ResultRegKind::OnlyGPRs); +} + +void BaseCompiler::needResultRegisters(ResultType type) { + needResultRegisters(type, ResultRegKind::All); +} + +void BaseCompiler::freeResultRegisters(ResultType type) { + freeResultRegisters(type, ResultRegKind::All); +} + +void BaseCompiler::captureResultRegisters(ResultType type) { + assertResultRegistersAvailable(type); + needResultRegisters(type); +} + +void BaseCompiler::captureCallResultRegisters(ResultType type) { + captureResultRegisters(type); +#ifdef JS_64BIT + widenInt32ResultRegisters(type); +#endif +} + +////////////////////////////////////////////////////////////////////////////// +// +// Control stack. Some of these are very hot. + +void BaseCompiler::initControl(Control& item, ResultType params) { + // Make sure the constructor was run properly + MOZ_ASSERT(!item.stackHeight.isValid() && item.stackSize == UINT32_MAX); + + uint32_t paramCount = deadCode_ ? 0 : params.length(); + uint32_t stackParamSize = stackConsumed(paramCount); + item.stackHeight = fr.stackResultsBase(stackParamSize); + item.stackSize = stk_.length() - paramCount; + item.deadOnArrival = deadCode_; + item.bceSafeOnEntry = bceSafe_; +} + +Control& BaseCompiler::controlItem() { return iter_.controlItem(); } + +Control& BaseCompiler::controlItem(uint32_t relativeDepth) { + return iter_.controlItem(relativeDepth); +} + +Control& BaseCompiler::controlOutermost() { return iter_.controlOutermost(); } + +LabelKind BaseCompiler::controlKind(uint32_t relativeDepth) { + return iter_.controlKind(relativeDepth); +} + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_reg_mgmt_inl_h diff --git a/js/src/wasm/WasmBCStk.h b/js/src/wasm/WasmBCStk.h new file mode 100644 index 0000000000..330e8abd06 --- /dev/null +++ b/js/src/wasm/WasmBCStk.h @@ -0,0 +1,345 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: Wasm value stack. + +#ifndef wasm_wasm_baseline_stk_h +#define wasm_wasm_baseline_stk_h + +#include "wasm/WasmBCDefs.h" +#include "wasm/WasmBCRegDefs.h" + +namespace js { +namespace wasm { + +// Value stack: stack elements + +struct Stk { + private: + Stk() : kind_(Unknown), i64val_(0) {} + + public: + enum Kind { + // The Mem opcodes are all clustered at the beginning to + // allow for a quick test within sync(). + MemI32, // 32-bit integer stack value ("offs") + MemI64, // 64-bit integer stack value ("offs") + MemF32, // 32-bit floating stack value ("offs") + MemF64, // 64-bit floating stack value ("offs") +#ifdef ENABLE_WASM_SIMD + MemV128, // 128-bit vector stack value ("offs") +#endif + MemRef, // reftype (pointer wide) stack value ("offs") + + // The Local opcodes follow the Mem opcodes for a similar + // quick test within hasLocal(). + LocalI32, // Local int32 var ("slot") + LocalI64, // Local int64 var ("slot") + LocalF32, // Local float32 var ("slot") + LocalF64, // Local double var ("slot") +#ifdef ENABLE_WASM_SIMD + LocalV128, // Local v128 var ("slot") +#endif + LocalRef, // Local reftype (pointer wide) var ("slot") + + RegisterI32, // 32-bit integer register ("i32reg") + RegisterI64, // 64-bit integer register ("i64reg") + RegisterF32, // 32-bit floating register ("f32reg") + RegisterF64, // 64-bit floating register ("f64reg") +#ifdef ENABLE_WASM_SIMD + RegisterV128, // 128-bit vector register ("v128reg") +#endif + RegisterRef, // reftype (pointer wide) register ("refReg") + + ConstI32, // 32-bit integer constant ("i32val") + ConstI64, // 64-bit integer constant ("i64val") + ConstF32, // 32-bit floating constant ("f32val") + ConstF64, // 64-bit floating constant ("f64val") +#ifdef ENABLE_WASM_SIMD + ConstV128, // 128-bit vector constant ("v128val") +#endif + ConstRef, // reftype (pointer wide) constant ("refval") + + Unknown, + }; + + Kind kind_; + + static const Kind MemLast = MemRef; + static const Kind LocalLast = LocalRef; + + union { + RegI32 i32reg_; + RegI64 i64reg_; + RegRef refReg_; + RegF32 f32reg_; + RegF64 f64reg_; +#ifdef ENABLE_WASM_SIMD + RegV128 v128reg_; +#endif + int32_t i32val_; + int64_t i64val_; + intptr_t refval_; + float f32val_; + double f64val_; +#ifdef ENABLE_WASM_SIMD + V128 v128val_; +#endif + uint32_t slot_; + uint32_t offs_; + }; + + explicit Stk(RegI32 r) : kind_(RegisterI32), i32reg_(r) {} + explicit Stk(RegI64 r) : kind_(RegisterI64), i64reg_(r) {} + explicit Stk(RegRef r) : kind_(RegisterRef), refReg_(r) {} + explicit Stk(RegF32 r) : kind_(RegisterF32), f32reg_(r) {} + explicit Stk(RegF64 r) : kind_(RegisterF64), f64reg_(r) {} +#ifdef ENABLE_WASM_SIMD + explicit Stk(RegV128 r) : kind_(RegisterV128), v128reg_(r) {} +#endif + explicit Stk(int32_t v) : kind_(ConstI32), i32val_(v) {} + explicit Stk(uint32_t v) : kind_(ConstI32), i32val_(int32_t(v)) {} + explicit Stk(int64_t v) : kind_(ConstI64), i64val_(v) {} + explicit Stk(float v) : kind_(ConstF32), f32val_(v) {} + explicit Stk(double v) : kind_(ConstF64), f64val_(v) {} +#ifdef ENABLE_WASM_SIMD + explicit Stk(V128 v) : kind_(ConstV128), v128val_(v) {} +#endif + explicit Stk(Kind k, uint32_t v) : kind_(k), slot_(v) { + MOZ_ASSERT(k > MemLast && k <= LocalLast); + } + static Stk StkRef(intptr_t v) { + Stk s; + s.kind_ = ConstRef; + s.refval_ = v; + return s; + } + static Stk StackResult(ValType type, uint32_t offs) { + Kind k; + switch (type.kind()) { + case ValType::I32: + k = Stk::MemI32; + break; + case ValType::I64: + k = Stk::MemI64; + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + k = Stk::MemV128; + break; +#else + MOZ_CRASH("No SIMD"); +#endif + case ValType::F32: + k = Stk::MemF32; + break; + case ValType::F64: + k = Stk::MemF64; + break; + case ValType::Ref: + k = Stk::MemRef; + break; + } + Stk s; + s.setOffs(k, offs); + return s; + } + + void setOffs(Kind k, uint32_t v) { + MOZ_ASSERT(k <= MemLast); + kind_ = k; + offs_ = v; + } + + Kind kind() const { return kind_; } + bool isMem() const { return kind_ <= MemLast; } + + RegI32 i32reg() const { + MOZ_ASSERT(kind_ == RegisterI32); + return i32reg_; + } + RegI64 i64reg() const { + MOZ_ASSERT(kind_ == RegisterI64); + return i64reg_; + } + RegRef refReg() const { + MOZ_ASSERT(kind_ == RegisterRef); + return refReg_; + } + RegF32 f32reg() const { + MOZ_ASSERT(kind_ == RegisterF32); + return f32reg_; + } + RegF64 f64reg() const { + MOZ_ASSERT(kind_ == RegisterF64); + return f64reg_; + } +#ifdef ENABLE_WASM_SIMD + RegV128 v128reg() const { + MOZ_ASSERT(kind_ == RegisterV128); + return v128reg_; + } +#endif + int32_t i32val() const { + MOZ_ASSERT(kind_ == ConstI32); + return i32val_; + } + int64_t i64val() const { + MOZ_ASSERT(kind_ == ConstI64); + return i64val_; + } + intptr_t refval() const { + MOZ_ASSERT(kind_ == ConstRef); + return refval_; + } + + // For these two, use an out-param instead of simply returning, to + // use the normal stack and not the x87 FP stack (which has effect on + // NaNs with the signaling bit set). + + void f32val(float* out) const { + MOZ_ASSERT(kind_ == ConstF32); + *out = f32val_; + } + void f64val(double* out) const { + MOZ_ASSERT(kind_ == ConstF64); + *out = f64val_; + } + +#ifdef ENABLE_WASM_SIMD + // For SIMD, do the same as for floats since we're using float registers to + // hold vectors; this is just conservative. + void v128val(V128* out) const { + MOZ_ASSERT(kind_ == ConstV128); + *out = v128val_; + } +#endif + + uint32_t slot() const { + MOZ_ASSERT(kind_ > MemLast && kind_ <= LocalLast); + return slot_; + } + uint32_t offs() const { + MOZ_ASSERT(isMem()); + return offs_; + } + +#ifdef DEBUG + // Print a stack element (Stk) to stderr. Skip the trailing \n. Printing + // of the actual contents of each stack element (see case ConstI32) can be + // filled in on demand -- even printing just the element `kind_` fields can + // be very useful. + void showStackElem() const { + switch (kind_) { + case MemI32: + fprintf(stderr, "MemI32()"); + break; + case MemI64: + fprintf(stderr, "MemI64()"); + break; + case MemF32: + fprintf(stderr, "MemF32()"); + break; + case MemF64: + fprintf(stderr, "MemF64()"); + break; +# ifdef ENABLE_WASM_SIMD + case MemV128: + fprintf(stderr, "MemV128()"); + break; +# endif + case MemRef: + fprintf(stderr, "MemRef()"); + break; + case LocalI32: + fprintf(stderr, "LocalI32()"); + break; + case LocalI64: + fprintf(stderr, "LocalI64()"); + break; + case LocalF32: + fprintf(stderr, "LocalF32()"); + break; + case LocalF64: + fprintf(stderr, "LocalF64()"); + break; +# ifdef ENABLE_WASM_SIMD + case LocalV128: + fprintf(stderr, "LocalV128()"); + break; +# endif + case LocalRef: + fprintf(stderr, "LocalRef()"); + break; + case RegisterI32: + fprintf(stderr, "RegisterI32()"); + break; + case RegisterI64: + fprintf(stderr, "RegisterI64()"); + break; + case RegisterF32: + fprintf(stderr, "RegisterF32()"); + break; + case RegisterF64: + fprintf(stderr, "RegisterF64()"); + break; +# ifdef ENABLE_WASM_SIMD + case RegisterV128: + fprintf(stderr, "RegisterV128()"); + break; +# endif + case RegisterRef: + fprintf(stderr, "RegisterRef()"); + break; + case ConstI32: + fprintf(stderr, "ConstI32(%d)", (int)i32val_); + break; + case ConstI64: + fprintf(stderr, "ConstI64()"); + break; + case ConstF32: + fprintf(stderr, "ConstF32()"); + break; + case ConstF64: + fprintf(stderr, "ConstF64()"); + break; +# ifdef ENABLE_WASM_SIMD + case ConstV128: + fprintf(stderr, "ConstV128()"); + break; +# endif + case ConstRef: + fprintf(stderr, "ConstRef()"); + break; + case Unknown: + fprintf(stderr, "Unknown()"); + break; + default: + fprintf(stderr, "!! Stk::showStackElem !!"); + break; + } + } +#endif +}; + +using StkVector = Vector; + +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_stk_h diff --git a/js/src/wasm/WasmBCStkMgmt-inl.h b/js/src/wasm/WasmBCStkMgmt-inl.h new file mode 100644 index 0000000000..6ce55cab3a --- /dev/null +++ b/js/src/wasm/WasmBCStkMgmt-inl.h @@ -0,0 +1,1320 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This is an INTERNAL header for Wasm baseline compiler: inline methods in the +// compiler for Stk values and value stack management. + +#ifndef wasm_wasm_baseline_stk_mgmt_inl_h +#define wasm_wasm_baseline_stk_mgmt_inl_h + +namespace js { +namespace wasm { + +#ifdef DEBUG +size_t BaseCompiler::countMemRefsOnStk() { + size_t nRefs = 0; + for (Stk& v : stk_) { + if (v.kind() == Stk::MemRef) { + nRefs++; + } + } + return nRefs; +} +#endif + +template +void BaseCompiler::push(T item) { + // None of the single-arg Stk constructors create a Stk::MemRef, so + // there's no need to increment stackMapGenerator_.memRefsOnStk here. + stk_.infallibleEmplaceBack(Stk(item)); +} + +void BaseCompiler::pushConstRef(intptr_t v) { + stk_.infallibleEmplaceBack(Stk::StkRef(v)); +} + +void BaseCompiler::loadConstI32(const Stk& src, RegI32 dest) { + moveImm32(src.i32val(), dest); +} + +void BaseCompiler::loadMemI32(const Stk& src, RegI32 dest) { + fr.loadStackI32(src.offs(), dest); +} + +void BaseCompiler::loadLocalI32(const Stk& src, RegI32 dest) { + fr.loadLocalI32(localFromSlot(src.slot(), MIRType::Int32), dest); +} + +void BaseCompiler::loadRegisterI32(const Stk& src, RegI32 dest) { + moveI32(src.i32reg(), dest); +} + +void BaseCompiler::loadConstI64(const Stk& src, RegI64 dest) { + moveImm64(src.i64val(), dest); +} + +void BaseCompiler::loadMemI64(const Stk& src, RegI64 dest) { + fr.loadStackI64(src.offs(), dest); +} + +void BaseCompiler::loadLocalI64(const Stk& src, RegI64 dest) { + fr.loadLocalI64(localFromSlot(src.slot(), MIRType::Int64), dest); +} + +void BaseCompiler::loadRegisterI64(const Stk& src, RegI64 dest) { + moveI64(src.i64reg(), dest); +} + +void BaseCompiler::loadConstRef(const Stk& src, RegRef dest) { + moveImmRef(src.refval(), dest); +} + +void BaseCompiler::loadMemRef(const Stk& src, RegRef dest) { + fr.loadStackRef(src.offs(), dest); +} + +void BaseCompiler::loadLocalRef(const Stk& src, RegRef dest) { + fr.loadLocalRef(localFromSlot(src.slot(), MIRType::RefOrNull), dest); +} + +void BaseCompiler::loadRegisterRef(const Stk& src, RegRef dest) { + moveRef(src.refReg(), dest); +} + +void BaseCompiler::loadConstF64(const Stk& src, RegF64 dest) { + double d; + src.f64val(&d); + masm.loadConstantDouble(d, dest); +} + +void BaseCompiler::loadMemF64(const Stk& src, RegF64 dest) { + fr.loadStackF64(src.offs(), dest); +} + +void BaseCompiler::loadLocalF64(const Stk& src, RegF64 dest) { + fr.loadLocalF64(localFromSlot(src.slot(), MIRType::Double), dest); +} + +void BaseCompiler::loadRegisterF64(const Stk& src, RegF64 dest) { + moveF64(src.f64reg(), dest); +} + +void BaseCompiler::loadConstF32(const Stk& src, RegF32 dest) { + float f; + src.f32val(&f); + masm.loadConstantFloat32(f, dest); +} + +void BaseCompiler::loadMemF32(const Stk& src, RegF32 dest) { + fr.loadStackF32(src.offs(), dest); +} + +void BaseCompiler::loadLocalF32(const Stk& src, RegF32 dest) { + fr.loadLocalF32(localFromSlot(src.slot(), MIRType::Float32), dest); +} + +void BaseCompiler::loadRegisterF32(const Stk& src, RegF32 dest) { + moveF32(src.f32reg(), dest); +} + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::loadConstV128(const Stk& src, RegV128 dest) { + V128 f; + src.v128val(&f); + masm.loadConstantSimd128(SimdConstant::CreateX16((int8_t*)f.bytes), dest); +} + +void BaseCompiler::loadMemV128(const Stk& src, RegV128 dest) { + fr.loadStackV128(src.offs(), dest); +} + +void BaseCompiler::loadLocalV128(const Stk& src, RegV128 dest) { + fr.loadLocalV128(localFromSlot(src.slot(), MIRType::Simd128), dest); +} + +void BaseCompiler::loadRegisterV128(const Stk& src, RegV128 dest) { + moveV128(src.v128reg(), dest); +} +#endif + +void BaseCompiler::loadI32(const Stk& src, RegI32 dest) { + switch (src.kind()) { + case Stk::ConstI32: + loadConstI32(src, dest); + break; + case Stk::MemI32: + loadMemI32(src, dest); + break; + case Stk::LocalI32: + loadLocalI32(src, dest); + break; + case Stk::RegisterI32: + loadRegisterI32(src, dest); + break; + default: + MOZ_CRASH("Compiler bug: Expected I32 on stack"); + } +} + +void BaseCompiler::loadI64(const Stk& src, RegI64 dest) { + switch (src.kind()) { + case Stk::ConstI64: + loadConstI64(src, dest); + break; + case Stk::MemI64: + loadMemI64(src, dest); + break; + case Stk::LocalI64: + loadLocalI64(src, dest); + break; + case Stk::RegisterI64: + loadRegisterI64(src, dest); + break; + default: + MOZ_CRASH("Compiler bug: Expected I64 on stack"); + } +} + +#if !defined(JS_PUNBOX64) +void BaseCompiler::loadI64Low(const Stk& src, RegI32 dest) { + switch (src.kind()) { + case Stk::ConstI64: + moveImm32(int32_t(src.i64val()), dest); + break; + case Stk::MemI64: + fr.loadStackI64Low(src.offs(), dest); + break; + case Stk::LocalI64: + fr.loadLocalI64Low(localFromSlot(src.slot(), MIRType::Int64), dest); + break; + case Stk::RegisterI64: + moveI32(RegI32(src.i64reg().low), dest); + break; + default: + MOZ_CRASH("Compiler bug: Expected I64 on stack"); + } +} + +void BaseCompiler::loadI64High(const Stk& src, RegI32 dest) { + switch (src.kind()) { + case Stk::ConstI64: + moveImm32(int32_t(src.i64val() >> 32), dest); + break; + case Stk::MemI64: + fr.loadStackI64High(src.offs(), dest); + break; + case Stk::LocalI64: + fr.loadLocalI64High(localFromSlot(src.slot(), MIRType::Int64), dest); + break; + case Stk::RegisterI64: + moveI32(RegI32(src.i64reg().high), dest); + break; + default: + MOZ_CRASH("Compiler bug: Expected I64 on stack"); + } +} +#endif + +void BaseCompiler::loadF64(const Stk& src, RegF64 dest) { + switch (src.kind()) { + case Stk::ConstF64: + loadConstF64(src, dest); + break; + case Stk::MemF64: + loadMemF64(src, dest); + break; + case Stk::LocalF64: + loadLocalF64(src, dest); + break; + case Stk::RegisterF64: + loadRegisterF64(src, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected F64 on stack"); + } +} + +void BaseCompiler::loadF32(const Stk& src, RegF32 dest) { + switch (src.kind()) { + case Stk::ConstF32: + loadConstF32(src, dest); + break; + case Stk::MemF32: + loadMemF32(src, dest); + break; + case Stk::LocalF32: + loadLocalF32(src, dest); + break; + case Stk::RegisterF32: + loadRegisterF32(src, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected F32 on stack"); + } +} + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::loadV128(const Stk& src, RegV128 dest) { + switch (src.kind()) { + case Stk::ConstV128: + loadConstV128(src, dest); + break; + case Stk::MemV128: + loadMemV128(src, dest); + break; + case Stk::LocalV128: + loadLocalV128(src, dest); + break; + case Stk::RegisterV128: + loadRegisterV128(src, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected V128 on stack"); + } +} +#endif + +void BaseCompiler::loadRef(const Stk& src, RegRef dest) { + switch (src.kind()) { + case Stk::ConstRef: + loadConstRef(src, dest); + break; + case Stk::MemRef: + loadMemRef(src, dest); + break; + case Stk::LocalRef: + loadLocalRef(src, dest); + break; + case Stk::RegisterRef: + loadRegisterRef(src, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected ref on stack"); + } +} + +void BaseCompiler::peekRefAt(uint32_t depth, RegRef dest) { + MOZ_ASSERT(depth < stk_.length()); + Stk& src = peek(stk_.length() - depth - 1); + loadRef(src, dest); +} + +// Flush all local and register value stack elements to memory. +// +// TODO / OPTIMIZE: As this is fairly expensive and causes worse +// code to be emitted subsequently, it is useful to avoid calling +// it. (Bug 1316802) +// +// Some optimization has been done already. Remaining +// opportunities: +// +// - It would be interesting to see if we can specialize it +// before calls with particularly simple signatures, or where +// we can do parallel assignment of register arguments, or +// similar. See notes in emitCall(). +// +// - Operations that need specific registers: multiply, quotient, +// remainder, will tend to sync because the registers we need +// will tend to be allocated. We may be able to avoid that by +// prioritizing registers differently (takeLast instead of +// takeFirst) but we may also be able to allocate an unused +// register on demand to free up one we need, thus avoiding the +// sync. That type of fix would go into needI32(). + +void BaseCompiler::sync() { + size_t start = 0; + size_t lim = stk_.length(); + + for (size_t i = lim; i > 0; i--) { + // Memory opcodes are first in the enum, single check against MemLast is + // fine. + if (stk_[i - 1].kind() <= Stk::MemLast) { + start = i; + break; + } + } + + for (size_t i = start; i < lim; i++) { + Stk& v = stk_[i]; + switch (v.kind()) { + case Stk::LocalI32: { + ScratchI32 scratch(*this); + loadLocalI32(v, scratch); + uint32_t offs = fr.pushGPR(scratch); + v.setOffs(Stk::MemI32, offs); + break; + } + case Stk::RegisterI32: { + uint32_t offs = fr.pushGPR(v.i32reg()); + freeI32(v.i32reg()); + v.setOffs(Stk::MemI32, offs); + break; + } + case Stk::LocalI64: { + ScratchI32 scratch(*this); +#ifdef JS_PUNBOX64 + loadI64(v, fromI32(scratch)); + uint32_t offs = fr.pushGPR(scratch); +#else + fr.loadLocalI64High(localFromSlot(v.slot(), MIRType::Int64), scratch); + fr.pushGPR(scratch); + fr.loadLocalI64Low(localFromSlot(v.slot(), MIRType::Int64), scratch); + uint32_t offs = fr.pushGPR(scratch); +#endif + v.setOffs(Stk::MemI64, offs); + break; + } + case Stk::RegisterI64: { +#ifdef JS_PUNBOX64 + uint32_t offs = fr.pushGPR(v.i64reg().reg); + freeI64(v.i64reg()); +#else + fr.pushGPR(v.i64reg().high); + uint32_t offs = fr.pushGPR(v.i64reg().low); + freeI64(v.i64reg()); +#endif + v.setOffs(Stk::MemI64, offs); + break; + } + case Stk::LocalF64: { + ScratchF64 scratch(*this); + loadF64(v, scratch); + uint32_t offs = fr.pushDouble(scratch); + v.setOffs(Stk::MemF64, offs); + break; + } + case Stk::RegisterF64: { + uint32_t offs = fr.pushDouble(v.f64reg()); + freeF64(v.f64reg()); + v.setOffs(Stk::MemF64, offs); + break; + } + case Stk::LocalF32: { + ScratchF32 scratch(*this); + loadF32(v, scratch); + uint32_t offs = fr.pushFloat32(scratch); + v.setOffs(Stk::MemF32, offs); + break; + } + case Stk::RegisterF32: { + uint32_t offs = fr.pushFloat32(v.f32reg()); + freeF32(v.f32reg()); + v.setOffs(Stk::MemF32, offs); + break; + } +#ifdef ENABLE_WASM_SIMD + case Stk::LocalV128: { + ScratchV128 scratch(*this); + loadV128(v, scratch); + uint32_t offs = fr.pushV128(scratch); + v.setOffs(Stk::MemV128, offs); + break; + } + case Stk::RegisterV128: { + uint32_t offs = fr.pushV128(v.v128reg()); + freeV128(v.v128reg()); + v.setOffs(Stk::MemV128, offs); + break; + } +#endif + case Stk::LocalRef: { + ScratchRef scratch(*this); + loadLocalRef(v, scratch); + uint32_t offs = fr.pushGPR(scratch); + v.setOffs(Stk::MemRef, offs); + stackMapGenerator_.memRefsOnStk++; + break; + } + case Stk::RegisterRef: { + uint32_t offs = fr.pushGPR(v.refReg()); + freeRef(v.refReg()); + v.setOffs(Stk::MemRef, offs); + stackMapGenerator_.memRefsOnStk++; + break; + } + default: { + break; + } + } + } +} + +// This is an optimization used to avoid calling sync() for +// setLocal(): if the local does not exist unresolved on the stack +// then we can skip the sync. + +bool BaseCompiler::hasLocal(uint32_t slot) { + for (size_t i = stk_.length(); i > 0; i--) { + // Memory opcodes are first in the enum, single check against MemLast is + // fine. + Stk::Kind kind = stk_[i - 1].kind(); + if (kind <= Stk::MemLast) { + return false; + } + + // Local opcodes follow memory opcodes in the enum, single check against + // LocalLast is sufficient. + if (kind <= Stk::LocalLast && stk_[i - 1].slot() == slot) { + return true; + } + } + return false; +} + +void BaseCompiler::syncLocal(uint32_t slot) { + if (hasLocal(slot)) { + sync(); // TODO / OPTIMIZE: Improve this? (Bug 1316817) + } +} + +// Push the register r onto the stack. + +void BaseCompiler::pushAny(AnyReg r) { + switch (r.tag) { + case AnyReg::I32: { + pushI32(r.i32()); + break; + } + case AnyReg::I64: { + pushI64(r.i64()); + break; + } + case AnyReg::F32: { + pushF32(r.f32()); + break; + } + case AnyReg::F64: { + pushF64(r.f64()); + break; + } +#ifdef ENABLE_WASM_SIMD + case AnyReg::V128: { + pushV128(r.v128()); + break; + } +#endif + case AnyReg::REF: { + pushRef(r.ref()); + break; + } + } +} + +void BaseCompiler::pushI32(RegI32 r) { + MOZ_ASSERT(!isAvailableI32(r)); + push(Stk(r)); +} + +void BaseCompiler::pushI64(RegI64 r) { + MOZ_ASSERT(!isAvailableI64(r)); + push(Stk(r)); +} + +void BaseCompiler::pushRef(RegRef r) { + MOZ_ASSERT(!isAvailableRef(r)); + push(Stk(r)); +} + +void BaseCompiler::pushPtr(RegPtr r) { + MOZ_ASSERT(!isAvailablePtr(r)); +#ifdef JS_64BIT + pushI64(RegI64(Register64(r))); +#else + pushI32(RegI32(r)); +#endif +} + +void BaseCompiler::pushF64(RegF64 r) { + MOZ_ASSERT(!isAvailableF64(r)); + push(Stk(r)); +} + +void BaseCompiler::pushF32(RegF32 r) { + MOZ_ASSERT(!isAvailableF32(r)); + push(Stk(r)); +} + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::pushV128(RegV128 r) { + MOZ_ASSERT(!isAvailableV128(r)); + push(Stk(r)); +} +#endif + +// Push the value onto the stack. PushI32 can also take uint32_t, and PushI64 +// can take uint64_t; the semantics are the same. Appropriate sign extension +// for a 32-bit value on a 64-bit architecture happens when the value is +// popped, see the definition of moveImm32 below. + +void BaseCompiler::pushI32(int32_t v) { push(Stk(v)); } + +void BaseCompiler::pushI64(int64_t v) { push(Stk(v)); } + +void BaseCompiler::pushRef(intptr_t v) { pushConstRef(v); } + +void BaseCompiler::pushPtr(intptr_t v) { +#ifdef JS_64BIT + pushI64(v); +#else + pushI32(v); +#endif +} + +void BaseCompiler::pushF64(double v) { push(Stk(v)); } + +void BaseCompiler::pushF32(float v) { push(Stk(v)); } + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::pushV128(V128 v) { push(Stk(v)); } +#endif + +// Push the local slot onto the stack. The slot will not be read +// here; it will be read when it is consumed, or when a side +// effect to the slot forces its value to be saved. + +void BaseCompiler::pushLocalI32(uint32_t slot) { + stk_.infallibleEmplaceBack(Stk(Stk::LocalI32, slot)); +} + +void BaseCompiler::pushLocalI64(uint32_t slot) { + stk_.infallibleEmplaceBack(Stk(Stk::LocalI64, slot)); +} + +void BaseCompiler::pushLocalRef(uint32_t slot) { + stk_.infallibleEmplaceBack(Stk(Stk::LocalRef, slot)); +} + +void BaseCompiler::pushLocalF64(uint32_t slot) { + stk_.infallibleEmplaceBack(Stk(Stk::LocalF64, slot)); +} + +void BaseCompiler::pushLocalF32(uint32_t slot) { + stk_.infallibleEmplaceBack(Stk(Stk::LocalF32, slot)); +} + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::pushLocalV128(uint32_t slot) { + stk_.infallibleEmplaceBack(Stk(Stk::LocalV128, slot)); +} +#endif + +void BaseCompiler::pushU32AsI64(RegI32 rs) { + RegI64 rd = widenI32(rs); + masm.move32To64ZeroExtend(rs, rd); + pushI64(rd); +} + +AnyReg BaseCompiler::popAny(AnyReg specific) { + switch (stk_.back().kind()) { + case Stk::MemI32: + case Stk::LocalI32: + case Stk::RegisterI32: + case Stk::ConstI32: + return AnyReg(popI32(specific.i32())); + + case Stk::MemI64: + case Stk::LocalI64: + case Stk::RegisterI64: + case Stk::ConstI64: + return AnyReg(popI64(specific.i64())); + + case Stk::MemF32: + case Stk::LocalF32: + case Stk::RegisterF32: + case Stk::ConstF32: + return AnyReg(popF32(specific.f32())); + + case Stk::MemF64: + case Stk::LocalF64: + case Stk::RegisterF64: + case Stk::ConstF64: + return AnyReg(popF64(specific.f64())); + +#ifdef ENABLE_WASM_SIMD + case Stk::MemV128: + case Stk::LocalV128: + case Stk::RegisterV128: + case Stk::ConstV128: + return AnyReg(popV128(specific.v128())); +#endif + + case Stk::MemRef: + case Stk::LocalRef: + case Stk::RegisterRef: + case Stk::ConstRef: + return AnyReg(popRef(specific.ref())); + + case Stk::Unknown: + MOZ_CRASH(); + + default: + MOZ_CRASH(); + } +} + +AnyReg BaseCompiler::popAny() { + switch (stk_.back().kind()) { + case Stk::MemI32: + case Stk::LocalI32: + case Stk::RegisterI32: + case Stk::ConstI32: + return AnyReg(popI32()); + + case Stk::MemI64: + case Stk::LocalI64: + case Stk::RegisterI64: + case Stk::ConstI64: + return AnyReg(popI64()); + + case Stk::MemF32: + case Stk::LocalF32: + case Stk::RegisterF32: + case Stk::ConstF32: + return AnyReg(popF32()); + + case Stk::MemF64: + case Stk::LocalF64: + case Stk::RegisterF64: + case Stk::ConstF64: + return AnyReg(popF64()); + +#ifdef ENABLE_WASM_SIMD + case Stk::MemV128: + case Stk::LocalV128: + case Stk::RegisterV128: + case Stk::ConstV128: + return AnyReg(popV128()); +#endif + + case Stk::MemRef: + case Stk::LocalRef: + case Stk::RegisterRef: + case Stk::ConstRef: + return AnyReg(popRef()); + + case Stk::Unknown: + MOZ_CRASH(); + + default: + MOZ_CRASH(); + } +} + +// Call only from other popI32() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popI32(const Stk& v, RegI32 dest) { + MOZ_ASSERT(&v == &stk_.back()); + switch (v.kind()) { + case Stk::ConstI32: + loadConstI32(v, dest); + break; + case Stk::LocalI32: + loadLocalI32(v, dest); + break; + case Stk::MemI32: + fr.popGPR(dest); + break; + case Stk::RegisterI32: + loadRegisterI32(v, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected int on stack"); + } +} + +RegI32 BaseCompiler::popI32() { + Stk& v = stk_.back(); + RegI32 r; + if (v.kind() == Stk::RegisterI32) { + r = v.i32reg(); + } else { + popI32(v, (r = needI32())); + } + stk_.popBack(); + return r; +} + +RegI32 BaseCompiler::popI32(RegI32 specific) { + Stk& v = stk_.back(); + + if (!(v.kind() == Stk::RegisterI32 && v.i32reg() == specific)) { + needI32(specific); + popI32(v, specific); + if (v.kind() == Stk::RegisterI32) { + freeI32(v.i32reg()); + } + } + + stk_.popBack(); + return specific; +} + +#ifdef ENABLE_WASM_SIMD +// Call only from other popV128() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popV128(const Stk& v, RegV128 dest) { + MOZ_ASSERT(&v == &stk_.back()); + switch (v.kind()) { + case Stk::ConstV128: + loadConstV128(v, dest); + break; + case Stk::LocalV128: + loadLocalV128(v, dest); + break; + case Stk::MemV128: + fr.popV128(dest); + break; + case Stk::RegisterV128: + loadRegisterV128(v, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected int on stack"); + } +} + +RegV128 BaseCompiler::popV128() { + Stk& v = stk_.back(); + RegV128 r; + if (v.kind() == Stk::RegisterV128) { + r = v.v128reg(); + } else { + popV128(v, (r = needV128())); + } + stk_.popBack(); + return r; +} + +RegV128 BaseCompiler::popV128(RegV128 specific) { + Stk& v = stk_.back(); + + if (!(v.kind() == Stk::RegisterV128 && v.v128reg() == specific)) { + needV128(specific); + popV128(v, specific); + if (v.kind() == Stk::RegisterV128) { + freeV128(v.v128reg()); + } + } + + stk_.popBack(); + return specific; +} +#endif + +// Call only from other popI64() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popI64(const Stk& v, RegI64 dest) { + MOZ_ASSERT(&v == &stk_.back()); + switch (v.kind()) { + case Stk::ConstI64: + loadConstI64(v, dest); + break; + case Stk::LocalI64: + loadLocalI64(v, dest); + break; + case Stk::MemI64: +#ifdef JS_PUNBOX64 + fr.popGPR(dest.reg); +#else + fr.popGPR(dest.low); + fr.popGPR(dest.high); +#endif + break; + case Stk::RegisterI64: + loadRegisterI64(v, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected long on stack"); + } +} + +RegI64 BaseCompiler::popI64() { + Stk& v = stk_.back(); + RegI64 r; + if (v.kind() == Stk::RegisterI64) { + r = v.i64reg(); + } else { + popI64(v, (r = needI64())); + } + stk_.popBack(); + return r; +} + +// Note, the stack top can be in one half of "specific" on 32-bit +// systems. We can optimize, but for simplicity, if the register +// does not match exactly, then just force the stack top to memory +// and then read it back in. + +RegI64 BaseCompiler::popI64(RegI64 specific) { + Stk& v = stk_.back(); + + if (!(v.kind() == Stk::RegisterI64 && v.i64reg() == specific)) { + needI64(specific); + popI64(v, specific); + if (v.kind() == Stk::RegisterI64) { + freeI64(v.i64reg()); + } + } + + stk_.popBack(); + return specific; +} + +// Call only from other popRef() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popRef(const Stk& v, RegRef dest) { + MOZ_ASSERT(&v == &stk_.back()); + switch (v.kind()) { + case Stk::ConstRef: + loadConstRef(v, dest); + break; + case Stk::LocalRef: + loadLocalRef(v, dest); + break; + case Stk::MemRef: + fr.popGPR(dest); + break; + case Stk::RegisterRef: + loadRegisterRef(v, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected ref on stack"); + } +} + +RegRef BaseCompiler::popRef(RegRef specific) { + Stk& v = stk_.back(); + + if (!(v.kind() == Stk::RegisterRef && v.refReg() == specific)) { + needRef(specific); + popRef(v, specific); + if (v.kind() == Stk::RegisterRef) { + freeRef(v.refReg()); + } + } + + stk_.popBack(); + if (v.kind() == Stk::MemRef) { + stackMapGenerator_.memRefsOnStk--; + } + return specific; +} + +RegRef BaseCompiler::popRef() { + Stk& v = stk_.back(); + RegRef r; + if (v.kind() == Stk::RegisterRef) { + r = v.refReg(); + } else { + popRef(v, (r = needRef())); + } + stk_.popBack(); + if (v.kind() == Stk::MemRef) { + stackMapGenerator_.memRefsOnStk--; + } + return r; +} + +// Call only from other popPtr() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popPtr(const Stk& v, RegPtr dest) { +#ifdef JS_64BIT + popI64(v, RegI64(Register64(dest))); +#else + popI32(v, RegI32(dest)); +#endif +} + +RegPtr BaseCompiler::popPtr(RegPtr specific) { +#ifdef JS_64BIT + return RegPtr(popI64(RegI64(Register64(specific))).reg); +#else + return RegPtr(popI32(RegI32(specific))); +#endif +} + +RegPtr BaseCompiler::popPtr() { +#ifdef JS_64BIT + return RegPtr(popI64().reg); +#else + return RegPtr(popI32()); +#endif +} + +// Call only from other popF64() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popF64(const Stk& v, RegF64 dest) { + MOZ_ASSERT(&v == &stk_.back()); + switch (v.kind()) { + case Stk::ConstF64: + loadConstF64(v, dest); + break; + case Stk::LocalF64: + loadLocalF64(v, dest); + break; + case Stk::MemF64: + fr.popDouble(dest); + break; + case Stk::RegisterF64: + loadRegisterF64(v, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected double on stack"); + } +} + +RegF64 BaseCompiler::popF64() { + Stk& v = stk_.back(); + RegF64 r; + if (v.kind() == Stk::RegisterF64) { + r = v.f64reg(); + } else { + popF64(v, (r = needF64())); + } + stk_.popBack(); + return r; +} + +RegF64 BaseCompiler::popF64(RegF64 specific) { + Stk& v = stk_.back(); + + if (!(v.kind() == Stk::RegisterF64 && v.f64reg() == specific)) { + needF64(specific); + popF64(v, specific); + if (v.kind() == Stk::RegisterF64) { + freeF64(v.f64reg()); + } + } + + stk_.popBack(); + return specific; +} + +// Call only from other popF32() variants. +// v must be the stack top. May pop the CPU stack. + +void BaseCompiler::popF32(const Stk& v, RegF32 dest) { + MOZ_ASSERT(&v == &stk_.back()); + switch (v.kind()) { + case Stk::ConstF32: + loadConstF32(v, dest); + break; + case Stk::LocalF32: + loadLocalF32(v, dest); + break; + case Stk::MemF32: + fr.popFloat32(dest); + break; + case Stk::RegisterF32: + loadRegisterF32(v, dest); + break; + default: + MOZ_CRASH("Compiler bug: expected float on stack"); + } +} + +RegF32 BaseCompiler::popF32() { + Stk& v = stk_.back(); + RegF32 r; + if (v.kind() == Stk::RegisterF32) { + r = v.f32reg(); + } else { + popF32(v, (r = needF32())); + } + stk_.popBack(); + return r; +} + +RegF32 BaseCompiler::popF32(RegF32 specific) { + Stk& v = stk_.back(); + + if (!(v.kind() == Stk::RegisterF32 && v.f32reg() == specific)) { + needF32(specific); + popF32(v, specific); + if (v.kind() == Stk::RegisterF32) { + freeF32(v.f32reg()); + } + } + + stk_.popBack(); + return specific; +} + +bool BaseCompiler::hasConst() const { + const Stk& v = stk_.back(); + switch (v.kind()) { + case Stk::ConstI32: + case Stk::ConstI64: + case Stk::ConstF32: + case Stk::ConstF64: +#ifdef ENABLE_WASM_SIMD + case Stk::ConstV128: +#endif + case Stk::ConstRef: + return true; + default: + return false; + } +} + +bool BaseCompiler::popConst(int32_t* c) { + Stk& v = stk_.back(); + if (v.kind() != Stk::ConstI32) { + return false; + } + *c = v.i32val(); + stk_.popBack(); + return true; +} + +bool BaseCompiler::popConst(int64_t* c) { + Stk& v = stk_.back(); + if (v.kind() != Stk::ConstI64) { + return false; + } + *c = v.i64val(); + stk_.popBack(); + return true; +} + +bool BaseCompiler::peekConst(int32_t* c) { + Stk& v = stk_.back(); + if (v.kind() != Stk::ConstI32) { + return false; + } + *c = v.i32val(); + return true; +} + +bool BaseCompiler::peekConst(int64_t* c) { + Stk& v = stk_.back(); + if (v.kind() != Stk::ConstI64) { + return false; + } + *c = v.i64val(); + return true; +} + +bool BaseCompiler::peek2xConst(int32_t* c0, int32_t* c1) { + MOZ_ASSERT(stk_.length() >= 2); + const Stk& v0 = *(stk_.end() - 1); + const Stk& v1 = *(stk_.end() - 2); + if (v0.kind() != Stk::ConstI32 || v1.kind() != Stk::ConstI32) { + return false; + } + *c0 = v0.i32val(); + *c1 = v1.i32val(); + return true; +} + +bool BaseCompiler::popConstPositivePowerOfTwo(int32_t* c, uint_fast8_t* power, + int32_t cutoff) { + Stk& v = stk_.back(); + if (v.kind() != Stk::ConstI32) { + return false; + } + *c = v.i32val(); + if (*c <= cutoff || !IsPowerOfTwo(static_cast(*c))) { + return false; + } + *power = FloorLog2(*c); + stk_.popBack(); + return true; +} + +bool BaseCompiler::popConstPositivePowerOfTwo(int64_t* c, uint_fast8_t* power, + int64_t cutoff) { + Stk& v = stk_.back(); + if (v.kind() != Stk::ConstI64) { + return false; + } + *c = v.i64val(); + if (*c <= cutoff || !IsPowerOfTwo(static_cast(*c))) { + return false; + } + *power = FloorLog2(*c); + stk_.popBack(); + return true; +} + +void BaseCompiler::pop2xI32(RegI32* r0, RegI32* r1) { + *r1 = popI32(); + *r0 = popI32(); +} + +void BaseCompiler::pop2xI64(RegI64* r0, RegI64* r1) { + *r1 = popI64(); + *r0 = popI64(); +} + +void BaseCompiler::pop2xF32(RegF32* r0, RegF32* r1) { + *r1 = popF32(); + *r0 = popF32(); +} + +void BaseCompiler::pop2xF64(RegF64* r0, RegF64* r1) { + *r1 = popF64(); + *r0 = popF64(); +} + +#ifdef ENABLE_WASM_SIMD +void BaseCompiler::pop2xV128(RegV128* r0, RegV128* r1) { + *r1 = popV128(); + *r0 = popV128(); +} +#endif + +void BaseCompiler::pop2xRef(RegRef* r0, RegRef* r1) { + *r1 = popRef(); + *r0 = popRef(); +} + +// Pop to a specific register +RegI32 BaseCompiler::popI32ToSpecific(RegI32 specific) { + freeI32(specific); + return popI32(specific); +} + +RegI64 BaseCompiler::popI64ToSpecific(RegI64 specific) { + freeI64(specific); + return popI64(specific); +} + +#ifdef JS_CODEGEN_ARM +// Pop an I64 as a valid register pair. +RegI64 BaseCompiler::popI64Pair() { + RegI64 r = needI64Pair(); + popI64ToSpecific(r); + return r; +} +#endif + +// Pop an I64 but narrow it and return the narrowed part. +RegI32 BaseCompiler::popI64ToI32() { + RegI64 r = popI64(); + return narrowI64(r); +} + +RegI32 BaseCompiler::popI64ToSpecificI32(RegI32 specific) { + RegI64 rd = widenI32(specific); + popI64ToSpecific(rd); + return narrowI64(rd); +} + +bool BaseCompiler::peekLocal(uint32_t* local) { + Stk& v = stk_.back(); + // See hasLocal() for documentation of this logic. + if (v.kind() <= Stk::MemLast || v.kind() > Stk::LocalLast) { + return false; + } + *local = v.slot(); + return true; +} + +size_t BaseCompiler::stackConsumed(size_t numval) { + size_t size = 0; + MOZ_ASSERT(numval <= stk_.length()); + for (uint32_t i = stk_.length() - 1; numval > 0; numval--, i--) { + Stk& v = stk_[i]; + switch (v.kind()) { + case Stk::MemRef: + size += BaseStackFrame::StackSizeOfPtr; + break; + case Stk::MemI32: + size += BaseStackFrame::StackSizeOfPtr; + break; + case Stk::MemI64: + size += BaseStackFrame::StackSizeOfInt64; + break; + case Stk::MemF64: + size += BaseStackFrame::StackSizeOfDouble; + break; + case Stk::MemF32: + size += BaseStackFrame::StackSizeOfFloat; + break; +#ifdef ENABLE_WASM_SIMD + case Stk::MemV128: + size += BaseStackFrame::StackSizeOfV128; + break; +#endif + default: + break; + } + } + return size; +} + +void BaseCompiler::popValueStackTo(uint32_t stackSize) { + for (uint32_t i = stk_.length(); i > stackSize; i--) { + Stk& v = stk_[i - 1]; + switch (v.kind()) { + case Stk::RegisterI32: + freeI32(v.i32reg()); + break; + case Stk::RegisterI64: + freeI64(v.i64reg()); + break; + case Stk::RegisterF64: + freeF64(v.f64reg()); + break; + case Stk::RegisterF32: + freeF32(v.f32reg()); + break; +#ifdef ENABLE_WASM_SIMD + case Stk::RegisterV128: + freeV128(v.v128reg()); + break; +#endif + case Stk::RegisterRef: + freeRef(v.refReg()); + break; + case Stk::MemRef: + stackMapGenerator_.memRefsOnStk--; + break; + default: + break; + } + } + stk_.shrinkTo(stackSize); +} + +void BaseCompiler::popValueStackBy(uint32_t items) { + popValueStackTo(stk_.length() - items); +} + +void BaseCompiler::dropValue() { + if (peek(0).isMem()) { + fr.popBytes(stackConsumed(1)); + } + popValueStackBy(1); +} + +// Peek at the stack, for calls. + +Stk& BaseCompiler::peek(uint32_t relativeDepth) { + return stk_[stk_.length() - 1 - relativeDepth]; +} +} // namespace wasm +} // namespace js + +#endif // wasm_wasm_baseline_stk_mgmt_inl_h diff --git a/js/src/wasm/WasmBaselineCompile.cpp b/js/src/wasm/WasmBaselineCompile.cpp new file mode 100644 index 0000000000..b277ca22c6 --- /dev/null +++ b/js/src/wasm/WasmBaselineCompile.cpp @@ -0,0 +1,11009 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * [SMDOC] WebAssembly baseline compiler (RabaldrMonkey) + * + * For now, see WasmBCClass.h for general comments about the compiler's + * structure. + * + * ---------------- + * + * General assumptions for 32-bit vs 64-bit code: + * + * - A 32-bit register can be extended in-place to a 64-bit register on 64-bit + * systems. + * + * - Code that knows that Register64 has a '.reg' member on 64-bit systems and + * '.high' and '.low' members on 32-bit systems, or knows the implications + * thereof, is #ifdef JS_PUNBOX64. All other code is #if(n)?def JS_64BIT. + * + * Coding standards are a little fluid: + * + * - In "small" code generating functions (eg emitMultiplyF64, emitQuotientI32, + * and surrounding functions; most functions fall into this class) where the + * meaning is obvious: + * + * Old school: + * - if there is a single source + destination register, it is called 'r' + * - if there is one source and a different destination, they are called 'rs' + * and 'rd' + * - if there is one source + destination register and another source register + * they are called 'r' and 'rs' + * - if there are two source registers and a destination register they are + * called 'rs0', 'rs1', and 'rd'. + * + * The new thing: + * - what is called 'r' in the old-school naming scheme is increasingly called + * 'rsd' in source+dest cases. + * + * - Generic temp registers are named /temp[0-9]?/ not /tmp[0-9]?/. + * + * - Registers can be named non-generically for their function ('rp' for the + * 'pointer' register and 'rv' for the 'value' register are typical) and those + * names may or may not have an 'r' prefix. + * + * - "Larger" code generating functions make their own rules. + */ + +#include "wasm/WasmBaselineCompile.h" + +#include "wasm/WasmBCClass.h" +#include "wasm/WasmBCDefs.h" +#include "wasm/WasmBCFrame.h" +#include "wasm/WasmBCRegDefs.h" +#include "wasm/WasmBCStk.h" + +#include "jit/MacroAssembler-inl.h" +#include "wasm/WasmBCClass-inl.h" +#include "wasm/WasmBCCodegen-inl.h" +#include "wasm/WasmBCRegDefs-inl.h" +#include "wasm/WasmBCRegMgmt-inl.h" +#include "wasm/WasmBCStkMgmt-inl.h" + +namespace js { +namespace wasm { + +using namespace js::jit; + +//////////////////////////////////////////////////////////// +// +// Out of line code management. + +// The baseline compiler will use OOL code more sparingly than Ion since our +// code is not high performance and frills like code density and branch +// prediction friendliness will be less important. +class OutOfLineCode : public TempObject { + private: + NonAssertingLabel entry_; + NonAssertingLabel rejoin_; + StackHeight stackHeight_; + + public: + OutOfLineCode() : stackHeight_(StackHeight::Invalid()) {} + + Label* entry() { return &entry_; } + Label* rejoin() { return &rejoin_; } + + void setStackHeight(StackHeight stackHeight) { + MOZ_ASSERT(!stackHeight_.isValid()); + stackHeight_ = stackHeight; + } + + void bind(BaseStackFrame* fr, MacroAssembler* masm) { + MOZ_ASSERT(stackHeight_.isValid()); + masm->bind(&entry_); + fr->setStackHeight(stackHeight_); + } + + // The generate() method must be careful about register use because it will be + // invoked when there is a register assignment in the BaseCompiler that does + // not correspond to the available registers when the generated OOL code is + // executed. The register allocator *must not* be called. + // + // The best strategy is for the creator of the OOL object to allocate all + // temps that the OOL code will need. + // + // Input, output, and temp registers are embedded in the OOL object and are + // known to the code generator. + // + // Scratch registers are available to use in OOL code. + // + // All other registers must be explicitly saved and restored by the OOL code + // before being used. + + virtual void generate(MacroAssembler* masm) = 0; +}; + +OutOfLineCode* BaseCompiler::addOutOfLineCode(OutOfLineCode* ool) { + if (!ool || !outOfLine_.append(ool)) { + return nullptr; + } + ool->setStackHeight(fr.stackHeight()); + return ool; +} + +bool BaseCompiler::generateOutOfLineCode() { + for (auto* ool : outOfLine_) { + if (!ool->entry()->used()) { + continue; + } + ool->bind(&fr, &masm); + ool->generate(&masm); + } + + return !masm.oom(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Sundry code generation. + +bool BaseCompiler::addInterruptCheck() { +#ifdef RABALDR_PIN_INSTANCE + Register tmp(InstanceReg); +#else + ScratchI32 tmp(*this); + fr.loadInstancePtr(tmp); +#endif + Label ok; + masm.branch32(Assembler::Equal, + Address(tmp, wasm::Instance::offsetOfInterrupt()), Imm32(0), + &ok); + masm.wasmTrap(wasm::Trap::CheckInterrupt, bytecodeOffset()); + masm.bind(&ok); + return createStackMap("addInterruptCheck"); +} + +void BaseCompiler::checkDivideByZero(RegI32 rhs) { + Label nonZero; + masm.branchTest32(Assembler::NonZero, rhs, rhs, &nonZero); + trap(Trap::IntegerDivideByZero); + masm.bind(&nonZero); +} + +void BaseCompiler::checkDivideByZero(RegI64 r) { + Label nonZero; + ScratchI32 scratch(*this); + masm.branchTest64(Assembler::NonZero, r, r, scratch, &nonZero); + trap(Trap::IntegerDivideByZero); + masm.bind(&nonZero); +} + +void BaseCompiler::checkDivideSignedOverflow(RegI32 rhs, RegI32 srcDest, + Label* done, bool zeroOnOverflow) { + Label notMin; + masm.branch32(Assembler::NotEqual, srcDest, Imm32(INT32_MIN), ¬Min); + if (zeroOnOverflow) { + masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), ¬Min); + moveImm32(0, srcDest); + masm.jump(done); + } else { + masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), ¬Min); + trap(Trap::IntegerOverflow); + } + masm.bind(¬Min); +} + +void BaseCompiler::checkDivideSignedOverflow(RegI64 rhs, RegI64 srcDest, + Label* done, bool zeroOnOverflow) { + Label notmin; + masm.branch64(Assembler::NotEqual, srcDest, Imm64(INT64_MIN), ¬min); + masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), ¬min); + if (zeroOnOverflow) { + masm.xor64(srcDest, srcDest); + masm.jump(done); + } else { + trap(Trap::IntegerOverflow); + } + masm.bind(¬min); +} + +void BaseCompiler::jumpTable(const LabelVector& labels, Label* theTable) { + // Flush constant pools to ensure that the table is never interrupted by + // constant pool entries. + masm.flush(); + +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + // Prevent nop sequences to appear in the jump table. + AutoForbidNops afn(&masm); +#endif + masm.bind(theTable); + + for (const auto& label : labels) { + CodeLabel cl; + masm.writeCodePointer(&cl); + cl.target()->bind(label.offset()); + masm.addCodeLabel(cl); + } +} + +void BaseCompiler::tableSwitch(Label* theTable, RegI32 switchValue, + Label* dispatchCode) { + masm.bind(dispatchCode); + +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) + ScratchI32 scratch(*this); + CodeLabel tableCl; + + masm.mov(&tableCl, scratch); + + tableCl.target()->bind(theTable->offset()); + masm.addCodeLabel(tableCl); + + masm.jmp(Operand(scratch, switchValue, ScalePointer)); +#elif defined(JS_CODEGEN_ARM) + // Flush constant pools: offset must reflect the distance from the MOV + // to the start of the table; as the address of the MOV is given by the + // label, nothing must come between the bind() and the ma_mov(). + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 5); + + ScratchI32 scratch(*this); + + // Compute the offset from the ma_mov instruction to the jump table. + Label here; + masm.bind(&here); + uint32_t offset = here.offset() - theTable->offset(); + + // Read PC+8 + masm.ma_mov(pc, scratch); + + // ARM scratch register is required by ma_sub. + ScratchRegisterScope arm_scratch(*this); + + // Compute the absolute table base pointer into `scratch`, offset by 8 + // to account for the fact that ma_mov read PC+8. + masm.ma_sub(Imm32(offset + 8), scratch, arm_scratch); + + // Jump indirect via table element. + masm.ma_ldr(DTRAddr(scratch, DtrRegImmShift(switchValue, LSL, 2)), pc, Offset, + Assembler::Always); +#elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) + ScratchI32 scratch(*this); + CodeLabel tableCl; + + masm.ma_li(scratch, &tableCl); + + tableCl.target()->bind(theTable->offset()); + masm.addCodeLabel(tableCl); + + masm.branchToComputedAddress(BaseIndex(scratch, switchValue, ScalePointer)); +#elif defined(JS_CODEGEN_ARM64) + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 4); + + ScratchI32 scratch(*this); + + ARMRegister s(scratch, 64); + ARMRegister v(switchValue, 64); + masm.Adr(s, theTable); + masm.Add(s, s, Operand(v, vixl::LSL, 3)); + masm.Ldr(s, MemOperand(s, 0)); + masm.Br(s); +#else + MOZ_CRASH("BaseCompiler platform hook: tableSwitch"); +#endif +} + +#ifdef JS_CODEGEN_X86 +void BaseCompiler::stashI64(RegPtr regForInstance, RegI64 r) { + MOZ_ASSERT(Instance::sizeOfBaselineScratch() >= 8); + MOZ_ASSERT(regForInstance != r.low && regForInstance != r.high); +# ifdef RABALDR_PIN_INSTANCE +# error "Pinned instance not expected" +# endif + fr.loadInstancePtr(regForInstance); + masm.store32(r.low, + Address(regForInstance, Instance::offsetOfBaselineScratch())); + masm.store32( + r.high, Address(regForInstance, Instance::offsetOfBaselineScratch() + 4)); +} + +void BaseCompiler::unstashI64(RegPtr regForInstance, RegI64 r) { + MOZ_ASSERT(Instance::sizeOfBaselineScratch() >= 8); +# ifdef RABALDR_PIN_INSTANCE +# error "Pinned instance not expected" +# endif + fr.loadInstancePtr(regForInstance); + if (regForInstance == r.low) { + masm.load32( + Address(regForInstance, Instance::offsetOfBaselineScratch() + 4), + r.high); + masm.load32(Address(regForInstance, Instance::offsetOfBaselineScratch()), + r.low); + } else { + masm.load32(Address(regForInstance, Instance::offsetOfBaselineScratch()), + r.low); + masm.load32( + Address(regForInstance, Instance::offsetOfBaselineScratch() + 4), + r.high); + } +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Function entry and exit + +bool BaseCompiler::beginFunction() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::beginFunction"); + + JitSpew(JitSpew_Codegen, "# ========================================"); + JitSpew(JitSpew_Codegen, "# Emitting wasm baseline code"); + JitSpew(JitSpew_Codegen, + "# beginFunction: start of function prologue for index %d", + (int)func_.index); + + // Make a start on the stackmap for this function. Inspect the args so + // as to determine which of them are both in-memory and pointer-typed, and + // add entries to machineStackTracker as appropriate. + + ArgTypeVector args(funcType()); + size_t inboundStackArgBytes = StackArgAreaSizeUnaligned(args); + MOZ_ASSERT(inboundStackArgBytes % sizeof(void*) == 0); + stackMapGenerator_.numStackArgWords = inboundStackArgBytes / sizeof(void*); + + MOZ_ASSERT(stackMapGenerator_.machineStackTracker.length() == 0); + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + stackMapGenerator_.numStackArgWords)) { + return false; + } + + // Identify GC-managed pointers passed on the stack. + for (WasmABIArgIter i(args); !i.done(); i++) { + ABIArg argLoc = *i; + if (argLoc.kind() == ABIArg::Stack && + args[i.index()] == MIRType::RefOrNull) { + uint32_t offset = argLoc.offsetFromArgBase(); + MOZ_ASSERT(offset < inboundStackArgBytes); + MOZ_ASSERT(offset % sizeof(void*) == 0); + stackMapGenerator_.machineStackTracker.setGCPointer(offset / + sizeof(void*)); + } + } + + GenerateFunctionPrologue( + masm, CallIndirectId::forFunc(moduleEnv_, func_.index), + compilerEnv_.mode() == CompileMode::Tier1 ? Some(func_.index) : Nothing(), + &offsets_); + + // GenerateFunctionPrologue pushes exactly one wasm::Frame's worth of + // stuff, and none of the values are GC pointers. Hence: + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + sizeof(Frame) / sizeof(void*))) { + return false; + } + + // Initialize DebugFrame fields before the stack overflow trap so that + // we have the invariant that all observable Frames in a debugEnabled + // Module have valid DebugFrames. + if (compilerEnv_.debugEnabled()) { +#ifdef JS_CODEGEN_ARM64 + static_assert(DebugFrame::offsetOfFrame() % WasmStackAlignment == 0, + "aligned"); +#endif + masm.reserveStack(DebugFrame::offsetOfFrame()); + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + DebugFrame::offsetOfFrame() / sizeof(void*))) { + return false; + } + + masm.store32(Imm32(func_.index), Address(masm.getStackPointer(), + DebugFrame::offsetOfFuncIndex())); + masm.store32(Imm32(0), + Address(masm.getStackPointer(), DebugFrame::offsetOfFlags())); + + // No need to initialize cachedReturnJSValue_ or any ref-typed spilled + // register results, as they are traced if and only if a corresponding + // flag (hasCachedReturnJSValue or hasSpilledRefRegisterResult) is set. + } + + // Generate a stack-overflow check and its associated stackmap. + + fr.checkStack(ABINonArgReg0, BytecodeOffset(func_.lineOrBytecode)); + + ExitStubMapVector extras; + if (!stackMapGenerator_.generateStackmapEntriesForTrapExit(args, &extras)) { + return false; + } + if (!createStackMap("stack check", extras, masm.currentOffset(), + HasDebugFrameWithLiveRefs::No)) { + return false; + } + + size_t reservedBytes = fr.fixedAllocSize() - masm.framePushed(); + MOZ_ASSERT(0 == (reservedBytes % sizeof(void*))); + + masm.reserveStack(reservedBytes); + fr.onFixedStackAllocated(); + if (!stackMapGenerator_.machineStackTracker.pushNonGCPointers( + reservedBytes / sizeof(void*))) { + return false; + } + + // Locals are stack allocated. Mark ref-typed ones in the stackmap + // accordingly. + for (const Local& l : localInfo_) { + // Locals that are stack arguments were already added to the stackmap + // before pushing the frame. + if (l.type == MIRType::RefOrNull && !l.isStackArgument()) { + uint32_t offs = fr.localOffsetFromSp(l); + MOZ_ASSERT(0 == (offs % sizeof(void*))); + stackMapGenerator_.machineStackTracker.setGCPointer(offs / sizeof(void*)); + } + } + + // Copy arguments from registers to stack. + for (WasmABIArgIter i(args); !i.done(); i++) { + if (args.isSyntheticStackResultPointerArg(i.index())) { + // If there are stack results and the pointer to stack results + // was passed in a register, store it to the stack. + if (i->argInRegister()) { + fr.storeIncomingStackResultAreaPtr(RegPtr(i->gpr())); + } + // If we're in a debug frame, copy the stack result pointer arg + // to a well-known place. + if (compilerEnv_.debugEnabled()) { + Register target = ABINonArgReturnReg0; + fr.loadIncomingStackResultAreaPtr(RegPtr(target)); + size_t debugFrameOffset = + masm.framePushed() - DebugFrame::offsetOfFrame(); + size_t debugStackResultsPointerOffset = + debugFrameOffset + DebugFrame::offsetOfStackResultsPointer(); + masm.storePtr(target, Address(masm.getStackPointer(), + debugStackResultsPointerOffset)); + } + continue; + } + if (!i->argInRegister()) { + continue; + } + Local& l = localInfo_[args.naturalIndex(i.index())]; + switch (i.mirType()) { + case MIRType::Int32: + fr.storeLocalI32(RegI32(i->gpr()), l); + break; + case MIRType::Int64: + fr.storeLocalI64(RegI64(i->gpr64()), l); + break; + case MIRType::RefOrNull: { + DebugOnly offs = fr.localOffsetFromSp(l); + MOZ_ASSERT(0 == (offs % sizeof(void*))); + fr.storeLocalRef(RegRef(i->gpr()), l); + // We should have just visited this local in the preceding loop. + MOZ_ASSERT(stackMapGenerator_.machineStackTracker.isGCPointer( + offs / sizeof(void*))); + break; + } + case MIRType::Double: + fr.storeLocalF64(RegF64(i->fpu()), l); + break; + case MIRType::Float32: + fr.storeLocalF32(RegF32(i->fpu()), l); + break; +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: + fr.storeLocalV128(RegV128(i->fpu()), l); + break; +#endif + default: + MOZ_CRASH("Function argument type"); + } + } + + fr.zeroLocals(&ra); + fr.storeInstancePtr(InstanceReg); + + if (compilerEnv_.debugEnabled()) { + insertBreakablePoint(CallSiteDesc::EnterFrame); + if (!createStackMap("debug: enter-frame breakpoint")) { + return false; + } + } + + JitSpew(JitSpew_Codegen, + "# beginFunction: enter body with masm.framePushed = %u", + masm.framePushed()); + MOZ_ASSERT(stackMapGenerator_.framePushedAtEntryToBody.isNothing()); + stackMapGenerator_.framePushedAtEntryToBody.emplace(masm.framePushed()); + + return true; +} + +bool BaseCompiler::endFunction() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::endFunction"); + + JitSpew(JitSpew_Codegen, "# endFunction: start of function epilogue"); + + // Always branch to returnLabel_. + masm.breakpoint(); + + // Patch the add in the prologue so that it checks against the correct + // frame size. Flush the constant pool in case it needs to be patched. + masm.flush(); + + // Precondition for patching. + if (masm.oom()) { + return false; + } + + fr.patchCheckStack(); + + masm.bind(&returnLabel_); + + ResultType resultType(ResultType::Vector(funcType().results())); + + popStackReturnValues(resultType); + + if (compilerEnv_.debugEnabled()) { + // Store and reload the return value from DebugFrame::return so that + // it can be clobbered, and/or modified by the debug trap. + saveRegisterReturnValues(resultType); + insertBreakablePoint(CallSiteDesc::Breakpoint); + if (!createStackMap("debug: return-point breakpoint", + HasDebugFrameWithLiveRefs::Maybe)) { + return false; + } + insertBreakablePoint(CallSiteDesc::LeaveFrame); + if (!createStackMap("debug: leave-frame breakpoint", + HasDebugFrameWithLiveRefs::Maybe)) { + return false; + } + restoreRegisterReturnValues(resultType); + } + +#ifndef RABALDR_PIN_INSTANCE + // To satisy instance extent invariant we need to reload InstanceReg because + // baseline can clobber it. + fr.loadInstancePtr(InstanceReg); +#endif + GenerateFunctionEpilogue(masm, fr.fixedAllocSize(), &offsets_); + +#if defined(JS_ION_PERF) + // FIXME - profiling code missing. No bug for this. + + // Note the end of the inline code and start of the OOL code. + // gen->perfSpewer().noteEndInlineCode(masm); +#endif + + JitSpew(JitSpew_Codegen, "# endFunction: end of function epilogue"); + JitSpew(JitSpew_Codegen, "# endFunction: start of OOL code"); + if (!generateOutOfLineCode()) { + return false; + } + JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code"); + + JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code"); + if (compilerEnv_.debugEnabled()) { + JitSpew(JitSpew_Codegen, "# endFunction: start of debug trap stub"); + insertBreakpointStub(); + JitSpew(JitSpew_Codegen, "# endFunction: end of debug trap stub"); + } + + offsets_.end = masm.currentOffset(); + + if (!fr.checkStackHeight()) { + return decoder_.fail(decoder_.beginOffset(), "stack frame is too large"); + } + + JitSpew(JitSpew_Codegen, "# endFunction: end of OOL code for index %d", + (int)func_.index); + return !masm.oom(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Debugger API. + +void BaseCompiler::insertBreakablePoint(CallSiteDesc::Kind kind) { +#ifndef RABALDR_PIN_INSTANCE + fr.loadInstancePtr(InstanceReg); +#endif + + // The breakpoint code must call the breakpoint handler installed on the + // instance if it is not null. There is one breakable point before + // every bytecode, and one at the beginning and at the end of the function. + // + // There are many constraints: + // + // - Code should be read-only; we do not want to patch + // - The breakpoint code should be as dense as possible, given the volume of + // breakable points + // - The handler-is-null case should be as fast as we can make it + // + // The scratch register is available here. + // + // An unconditional callout would be densest but is too slow. The best + // balance results from an inline test for null with a conditional call. The + // best code sequence is platform-dependent. + // + // The conditional call goes to a stub attached to the function that performs + // further filtering before calling the breakpoint handler. +#if defined(JS_CODEGEN_X64) + // REX 83 MODRM OFFS IB + static_assert(Instance::offsetOfDebugTrapHandler() < 128); + masm.cmpq(Imm32(0), Operand(Address(InstanceReg, + Instance::offsetOfDebugTrapHandler()))); + + // 74 OFFS + Label L; + L.bind(masm.currentOffset() + 7); + masm.j(Assembler::Zero, &L); + + // E8 OFFS OFFS OFFS OFFS + masm.call(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + + // Branch destination + MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() == uint32_t(L.offset())); +#elif defined(JS_CODEGEN_X86) + // 83 MODRM OFFS IB + static_assert(Instance::offsetOfDebugTrapHandler() < 128); + masm.cmpl(Imm32(0), Operand(Address(InstanceReg, + Instance::offsetOfDebugTrapHandler()))); + + // 74 OFFS + Label L; + L.bind(masm.currentOffset() + 7); + masm.j(Assembler::Zero, &L); + + // E8 OFFS OFFS OFFS OFFS + masm.call(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + + // Branch destination + MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() == uint32_t(L.offset())); +#elif defined(JS_CODEGEN_ARM64) + ScratchPtr scratch(*this); + ARMRegister tmp(scratch, 64); + Label L; + masm.Ldr(tmp, MemOperand(Address(InstanceReg, + Instance::offsetOfDebugTrapHandler()))); + masm.Cbz(tmp, &L); + masm.Bl(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + masm.bind(&L); +#elif defined(JS_CODEGEN_ARM) + ScratchPtr scratch(*this); + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()), + scratch); + masm.ma_orr(scratch, scratch, SetCC); + masm.ma_bl(&debugTrapStub_, Assembler::NonZero); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); +#elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_RISCV64) + ScratchPtr scratch(*this); + Label L; + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugTrapHandler()), + scratch); + masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), &L); + masm.call(&debugTrapStub_); + masm.append(CallSiteDesc(iter_.lastOpcodeOffset(), kind), + CodeOffset(masm.currentOffset())); + masm.bind(&L); +#else + MOZ_CRASH("BaseCompiler platform hook: insertBreakablePoint"); +#endif +} + +void BaseCompiler::insertBreakpointStub() { + // The debug trap stub performs out-of-line filtering before jumping to the + // debug trap handler if necessary. The trap handler returns directly to + // the breakable point. + // + // NOTE, the link register is live here on platforms that have LR. + // + // The scratch register is available here (as it was at the call site). + // + // It's useful for the debug trap stub to be compact, as every function gets + // one. + + Label L; + masm.bind(&debugTrapStub_); + +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + { + ScratchPtr scratch(*this); + + // Get the per-instance table of filtering bits. + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()), + scratch); + + // Check the filter bit. There is one bit per function in the module. + // Table elements are 32-bit because the masm makes that convenient. + masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32), + Imm32(1 << (func_.index % 32)), &L); + + // Fast path: return to the execution. + masm.ret(); + } +#elif defined(JS_CODEGEN_ARM64) + { + ScratchPtr scratch(*this); + + // Logic as above, except abiret to jump to the LR directly + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()), + scratch); + masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32), + Imm32(1 << (func_.index % 32)), &L); + masm.abiret(); + } +#elif defined(JS_CODEGEN_ARM) + { + // We must be careful not to use the SecondScratchRegister, which usually + // is LR, as LR is live here. This means avoiding masm abstractions such + // as branchTest32. + + static_assert(ScratchRegister != lr); + static_assert(Instance::offsetOfDebugFilter() < 0x1000); + + ScratchRegisterScope tmp1(masm); + ScratchI32 tmp2(*this); + masm.ma_ldr( + DTRAddr(InstanceReg, DtrOffImm(Instance::offsetOfDebugFilter())), tmp1); + masm.ma_mov(Imm32(func_.index / 32), tmp2); + masm.ma_ldr(DTRAddr(tmp1, DtrRegImmShift(tmp2, LSL, 0)), tmp2); + masm.ma_tst(tmp2, Imm32(1 << func_.index % 32), tmp1, Assembler::Always); + masm.ma_bx(lr, Assembler::Zero); + } +#elif defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_RISCV64) + { + ScratchPtr scratch(*this); + + // Logic same as ARM64. + masm.loadPtr(Address(InstanceReg, Instance::offsetOfDebugFilter()), + scratch); + masm.branchTest32(Assembler::NonZero, Address(scratch, func_.index / 32), + Imm32(1 << (func_.index % 32)), &L); + masm.abiret(); + } +#else + MOZ_CRASH("BaseCompiler platform hook: endFunction"); +#endif + + // Jump to the debug trap handler. + masm.bind(&L); + masm.jump(Address(InstanceReg, Instance::offsetOfDebugTrapHandler())); +} + +void BaseCompiler::saveRegisterReturnValues(const ResultType& resultType) { + MOZ_ASSERT(compilerEnv_.debugEnabled()); + size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame(); + size_t registerResultIdx = 0; + for (ABIResultIter i(resultType); !i.done(); i.next()) { + const ABIResult result = i.cur(); + if (!result.inRegister()) { +#ifdef DEBUG + for (i.next(); !i.done(); i.next()) { + MOZ_ASSERT(!i.cur().inRegister()); + } +#endif + break; + } + + size_t resultOffset = DebugFrame::offsetOfRegisterResult(registerResultIdx); + Address dest(masm.getStackPointer(), debugFrameOffset + resultOffset); + switch (result.type().kind()) { + case ValType::I32: + masm.store32(RegI32(result.gpr()), dest); + break; + case ValType::I64: + masm.store64(RegI64(result.gpr64()), dest); + break; + case ValType::F64: + masm.storeDouble(RegF64(result.fpr()), dest); + break; + case ValType::F32: + masm.storeFloat32(RegF32(result.fpr()), dest); + break; + case ValType::Ref: { + uint32_t flag = + DebugFrame::hasSpilledRegisterRefResultBitMask(registerResultIdx); + // Tell Instance::traceFrame that we have a pointer to trace. + masm.or32(Imm32(flag), + Address(masm.getStackPointer(), + debugFrameOffset + DebugFrame::offsetOfFlags())); + masm.storePtr(RegRef(result.gpr()), dest); + break; + } + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + masm.storeUnalignedSimd128(RegV128(result.fpr()), dest); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + registerResultIdx++; + } +} + +void BaseCompiler::restoreRegisterReturnValues(const ResultType& resultType) { + MOZ_ASSERT(compilerEnv_.debugEnabled()); + size_t debugFrameOffset = masm.framePushed() - DebugFrame::offsetOfFrame(); + size_t registerResultIdx = 0; + for (ABIResultIter i(resultType); !i.done(); i.next()) { + const ABIResult result = i.cur(); + if (!result.inRegister()) { +#ifdef DEBUG + for (i.next(); !i.done(); i.next()) { + MOZ_ASSERT(!i.cur().inRegister()); + } +#endif + break; + } + size_t resultOffset = + DebugFrame::offsetOfRegisterResult(registerResultIdx++); + Address src(masm.getStackPointer(), debugFrameOffset + resultOffset); + switch (result.type().kind()) { + case ValType::I32: + masm.load32(src, RegI32(result.gpr())); + break; + case ValType::I64: + masm.load64(src, RegI64(result.gpr64())); + break; + case ValType::F64: + masm.loadDouble(src, RegF64(result.fpr())); + break; + case ValType::F32: + masm.loadFloat32(src, RegF32(result.fpr())); + break; + case ValType::Ref: + masm.loadPtr(src, RegRef(result.gpr())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + masm.loadUnalignedSimd128(src, RegV128(result.fpr())); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + } +} + +////////////////////////////////////////////////////////////////////////////// +// +// Results and block parameters + +void BaseCompiler::popStackReturnValues(const ResultType& resultType) { + uint32_t bytes = ABIResultIter::MeasureStackBytes(resultType); + if (bytes == 0) { + return; + } + Register target = ABINonArgReturnReg0; + Register temp = ABINonArgReturnReg1; + fr.loadIncomingStackResultAreaPtr(RegPtr(target)); + fr.popStackResultsToMemory(target, bytes, temp); +} + +// TODO / OPTIMIZE (Bug 1316818): At the moment we use the Wasm +// inter-procedure ABI for block returns, which allocates ReturnReg as the +// single block result register. It is possible other choices would lead to +// better register allocation, as ReturnReg is often first in the register set +// and will be heavily wanted by the register allocator that uses takeFirst(). +// +// Obvious options: +// - pick a register at the back of the register set +// - pick a random register per block (different blocks have +// different join regs) + +void BaseCompiler::popRegisterResults(ABIResultIter& iter) { + // Pop register results. Note that in the single-value case, popping to a + // register may cause a sync(); for multi-value we sync'd already. + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + if (!result.inRegister()) { + // TODO / OPTIMIZE: We sync here to avoid solving the general parallel + // move problem in popStackResults. However we could avoid syncing the + // values that are going to registers anyway, if they are already in + // registers. + sync(); + break; + } + switch (result.type().kind()) { + case ValType::I32: + popI32(RegI32(result.gpr())); + break; + case ValType::I64: + popI64(RegI64(result.gpr64())); + break; + case ValType::F32: + popF32(RegF32(result.fpr())); + break; + case ValType::F64: + popF64(RegF64(result.fpr())); + break; + case ValType::Ref: + popRef(RegRef(result.gpr())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + popV128(RegV128(result.fpr())); +#else + MOZ_CRASH("No SIMD support"); +#endif + } + } +} + +void BaseCompiler::popStackResults(ABIResultIter& iter, StackHeight stackBase) { + MOZ_ASSERT(!iter.done()); + + // The iterator should be advanced beyond register results, and register + // results should be popped already from the value stack. + uint32_t alreadyPopped = iter.index(); + + // At this point, only stack arguments are remaining. Iterate through them + // to measure how much stack space they will take up. + for (; !iter.done(); iter.next()) { + MOZ_ASSERT(iter.cur().onStack()); + } + + // Calculate the space needed to store stack results, in bytes. + uint32_t stackResultBytes = iter.stackBytesConsumedSoFar(); + MOZ_ASSERT(stackResultBytes); + + // Compute the stack height including the stack results. Note that it's + // possible that this call expands the stack, for example if some of the + // results are supplied by constants and so are not already on the machine + // stack. + uint32_t endHeight = fr.prepareStackResultArea(stackBase, stackResultBytes); + + // Find a free GPR to use when shuffling stack values. If none is + // available, push ReturnReg and restore it after we're done. + bool saved = false; + RegPtr temp = ra.needTempPtr(RegPtr(ReturnReg), &saved); + + // The sequence of Stk values is in the same order on the machine stack as + // the result locations, but there is a complication: constant values are + // not actually pushed on the machine stack. (At this point registers and + // locals have been spilled already.) So, moving the Stk values into place + // isn't simply a shuffle-down or shuffle-up operation. There is a part of + // the Stk sequence that shuffles toward the FP, a part that's already in + // place, and a part that shuffles toward the SP. After shuffling, we have + // to materialize the constants. + + // Shuffle mem values toward the frame pointer, copying deepest values + // first. Stop when we run out of results, get to a register result, or + // find a Stk value that is closer to the FP than the result. + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + if (!result.onStack()) { + break; + } + MOZ_ASSERT(result.stackOffset() < stackResultBytes); + uint32_t destHeight = endHeight - result.stackOffset(); + uint32_t stkBase = stk_.length() - (iter.count() - alreadyPopped); + Stk& v = stk_[stkBase + iter.index()]; + if (v.isMem()) { + uint32_t srcHeight = v.offs(); + if (srcHeight <= destHeight) { + break; + } + fr.shuffleStackResultsTowardFP(srcHeight, destHeight, result.size(), + temp); + } + } + + // Reset iterator and skip register results. + for (iter.reset(); !iter.done(); iter.next()) { + if (iter.cur().onStack()) { + break; + } + } + + // Revisit top stack values, shuffling mem values toward the stack pointer, + // copying shallowest values first. + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + MOZ_ASSERT(result.onStack()); + MOZ_ASSERT(result.stackOffset() < stackResultBytes); + uint32_t destHeight = endHeight - result.stackOffset(); + Stk& v = stk_[stk_.length() - (iter.index() - alreadyPopped) - 1]; + if (v.isMem()) { + uint32_t srcHeight = v.offs(); + if (srcHeight >= destHeight) { + break; + } + fr.shuffleStackResultsTowardSP(srcHeight, destHeight, result.size(), + temp); + } + } + + // Reset iterator and skip register results, which are already popped off + // the value stack. + for (iter.reset(); !iter.done(); iter.next()) { + if (iter.cur().onStack()) { + break; + } + } + + // Materialize constants and pop the remaining items from the value stack. + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + uint32_t resultHeight = endHeight - result.stackOffset(); + Stk& v = stk_.back(); + switch (v.kind()) { + case Stk::ConstI32: +#if defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) + fr.storeImmediatePtrToStack(v.i32val_, resultHeight, temp); +#else + fr.storeImmediatePtrToStack(uint32_t(v.i32val_), resultHeight, temp); +#endif + break; + case Stk::ConstF32: + fr.storeImmediateF32ToStack(v.f32val_, resultHeight, temp); + break; + case Stk::ConstI64: + fr.storeImmediateI64ToStack(v.i64val_, resultHeight, temp); + break; + case Stk::ConstF64: + fr.storeImmediateF64ToStack(v.f64val_, resultHeight, temp); + break; +#ifdef ENABLE_WASM_SIMD + case Stk::ConstV128: + fr.storeImmediateV128ToStack(v.v128val_, resultHeight, temp); + break; +#endif + case Stk::ConstRef: + fr.storeImmediatePtrToStack(v.refval_, resultHeight, temp); + break; + case Stk::MemRef: + // Update bookkeeping as we pop the Stk entry. + stackMapGenerator_.memRefsOnStk--; + break; + default: + MOZ_ASSERT(v.isMem()); + break; + } + stk_.popBack(); + } + + ra.freeTempPtr(temp, saved); + + // This will pop the stack if needed. + fr.finishStackResultArea(stackBase, stackResultBytes); +} + +void BaseCompiler::popBlockResults(ResultType type, StackHeight stackBase, + ContinuationKind kind) { + if (!type.empty()) { + ABIResultIter iter(type); + popRegisterResults(iter); + if (!iter.done()) { + popStackResults(iter, stackBase); + // Because popStackResults might clobber the stack, it leaves the stack + // pointer already in the right place for the continuation, whether the + // continuation is a jump or fallthrough. + return; + } + } + // We get here if there are no stack results. For a fallthrough, the stack + // is already at the right height. For a jump, we may need to pop the stack + // pointer if the continuation's stack height is lower than the current + // stack height. + if (kind == ContinuationKind::Jump) { + fr.popStackBeforeBranch(stackBase, type); + } +} + +// This function is similar to popBlockResults, but additionally handles the +// implicit exception pointer that is pushed to the value stack on entry to +// a catch handler by dropping it appropriately. +void BaseCompiler::popCatchResults(ResultType type, StackHeight stackBase) { + if (!type.empty()) { + ABIResultIter iter(type); + popRegisterResults(iter); + if (!iter.done()) { + popStackResults(iter, stackBase); + // Since popStackResults clobbers the stack, we only need to free the + // exception off of the value stack. + popValueStackBy(1); + } else { + // If there are no stack results, we have to adjust the stack by + // dropping the exception reference that's now on the stack. + dropValue(); + } + } else { + dropValue(); + } + fr.popStackBeforeBranch(stackBase, type); +} + +Stk BaseCompiler::captureStackResult(const ABIResult& result, + StackHeight resultsBase, + uint32_t stackResultBytes) { + MOZ_ASSERT(result.onStack()); + uint32_t offs = fr.locateStackResult(result, resultsBase, stackResultBytes); + return Stk::StackResult(result.type(), offs); +} + +// TODO: It may be fruitful to inline the fast path here, as it will be common. + +bool BaseCompiler::pushResults(ResultType type, StackHeight resultsBase) { + if (type.empty()) { + return true; + } + + if (type.length() > 1) { + // Reserve extra space on the stack for all the values we'll push. + // Multi-value push is not accounted for by the pre-sizing of the stack in + // the decoding loop. + // + // Also make sure we leave headroom for other pushes that will occur after + // pushing results, just to be safe. + if (!stk_.reserve(stk_.length() + type.length() + MaxPushesPerOpcode)) { + return false; + } + } + + // We need to push the results in reverse order, so first iterate through + // all results to determine the locations of stack result types. + ABIResultIter iter(type); + while (!iter.done()) { + iter.next(); + } + uint32_t stackResultBytes = iter.stackBytesConsumedSoFar(); + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + if (!result.onStack()) { + break; + } + Stk v = captureStackResult(result, resultsBase, stackResultBytes); + push(v); + if (v.kind() == Stk::MemRef) { + stackMapGenerator_.memRefsOnStk++; + } + } + + for (; !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + MOZ_ASSERT(result.inRegister()); + switch (result.type().kind()) { + case ValType::I32: + pushI32(RegI32(result.gpr())); + break; + case ValType::I64: + pushI64(RegI64(result.gpr64())); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + pushV128(RegV128(result.fpr())); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + case ValType::F32: + pushF32(RegF32(result.fpr())); + break; + case ValType::F64: + pushF64(RegF64(result.fpr())); + break; + case ValType::Ref: + pushRef(RegRef(result.gpr())); + break; + } + } + + return true; +} + +bool BaseCompiler::pushBlockResults(ResultType type) { + return pushResults(type, controlItem().stackHeight); +} + +// A combination of popBlockResults + pushBlockResults, used when entering a +// block with a control-flow join (loops) or split (if) to shuffle the +// fallthrough block parameters into the locations expected by the +// continuation. +bool BaseCompiler::topBlockParams(ResultType type) { + // This function should only be called when entering a block with a + // control-flow join at the entry, where there are no live temporaries in + // the current block. + StackHeight base = controlItem().stackHeight; + MOZ_ASSERT(fr.stackResultsBase(stackConsumed(type.length())) == base); + popBlockResults(type, base, ContinuationKind::Fallthrough); + return pushBlockResults(type); +} + +// A combination of popBlockResults + pushBlockResults, used before branches +// where we don't know the target (br_if / br_table). If and when the branch +// is taken, the stack results will be shuffled down into place. For br_if +// that has fallthrough, the parameters for the untaken branch flow through to +// the continuation. +bool BaseCompiler::topBranchParams(ResultType type, StackHeight* height) { + if (type.empty()) { + *height = fr.stackHeight(); + return true; + } + // There may be temporary values that need spilling; delay computation of + // the stack results base until after the popRegisterResults(), which spills + // if needed. + ABIResultIter iter(type); + popRegisterResults(iter); + StackHeight base = fr.stackResultsBase(stackConsumed(iter.remaining())); + if (!iter.done()) { + popStackResults(iter, base); + } + if (!pushResults(type, base)) { + return false; + } + *height = base; + return true; +} + +// Conditional branches with fallthrough are preceded by a topBranchParams, so +// we know that there are no stack results that need to be materialized. In +// that case, we can just shuffle the whole block down before popping the +// stack. +void BaseCompiler::shuffleStackResultsBeforeBranch(StackHeight srcHeight, + StackHeight destHeight, + ResultType type) { + uint32_t stackResultBytes = 0; + + if (ABIResultIter::HasStackResults(type)) { + MOZ_ASSERT(stk_.length() >= type.length()); + ABIResultIter iter(type); + for (; !iter.done(); iter.next()) { +#ifdef DEBUG + const ABIResult& result = iter.cur(); + const Stk& v = stk_[stk_.length() - iter.index() - 1]; + MOZ_ASSERT(v.isMem() == result.onStack()); +#endif + } + + stackResultBytes = iter.stackBytesConsumedSoFar(); + MOZ_ASSERT(stackResultBytes > 0); + + if (srcHeight != destHeight) { + // Find a free GPR to use when shuffling stack values. If none + // is available, push ReturnReg and restore it after we're done. + bool saved = false; + RegPtr temp = ra.needTempPtr(RegPtr(ReturnReg), &saved); + fr.shuffleStackResultsTowardFP(srcHeight, destHeight, stackResultBytes, + temp); + ra.freeTempPtr(temp, saved); + } + } + + fr.popStackBeforeBranch(destHeight, stackResultBytes); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Function calls. + +void BaseCompiler::beginCall( + FunctionCall& call, UseABI useABI, + RestoreRegisterStateAndRealm restoreRegisterStateAndRealm) { + MOZ_ASSERT_IF( + useABI == UseABI::Builtin, + restoreRegisterStateAndRealm == RestoreRegisterStateAndRealm::False); + + call.restoreRegisterStateAndRealm = + restoreRegisterStateAndRealm == RestoreRegisterStateAndRealm::True; + call.usesSystemAbi = useABI == UseABI::System; + + if (call.usesSystemAbi) { + // Call-outs need to use the appropriate system ABI. +#if defined(JS_CODEGEN_ARM) + call.hardFP = UseHardFpABI(); + call.abi.setUseHardFp(call.hardFP); +#endif + } else { +#if defined(JS_CODEGEN_ARM) + MOZ_ASSERT(call.hardFP, "All private ABIs pass FP arguments in registers"); +#endif + } + + // Use masm.framePushed() because the value we want here does not depend + // on the height of the frame's stack area, but the actual size of the + // allocated frame. + call.frameAlignAdjustment = ComputeByteAlignment( + masm.framePushed() + sizeof(Frame), JitStackAlignment); +} + +void BaseCompiler::endCall(FunctionCall& call, size_t stackSpace) { + size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment; + fr.freeArgAreaAndPopBytes(adjustment, stackSpace); + + MOZ_ASSERT(stackMapGenerator_.framePushedExcludingOutboundCallArgs.isSome()); + stackMapGenerator_.framePushedExcludingOutboundCallArgs.reset(); + + if (call.restoreRegisterStateAndRealm) { + // The instance has been clobbered, so always reload + fr.loadInstancePtr(InstanceReg); + masm.loadWasmPinnedRegsFromInstance(); + masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1); + } else if (call.usesSystemAbi) { + // On x86 there are no pinned registers, so don't waste time + // reloading the instance. +#ifndef JS_CODEGEN_X86 + // The instance has been clobbered, so always reload + fr.loadInstancePtr(InstanceReg); + masm.loadWasmPinnedRegsFromInstance(); +#endif + } +} + +void BaseCompiler::startCallArgs(size_t stackArgAreaSizeUnaligned, + FunctionCall* call) { + size_t stackArgAreaSizeAligned = + AlignStackArgAreaSize(stackArgAreaSizeUnaligned); + MOZ_ASSERT(stackArgAreaSizeUnaligned <= stackArgAreaSizeAligned); + + // Record the masm.framePushed() value at this point, before we push args + // for the call, but including the alignment space placed above the args. + // This defines the lower limit of the stackmap that will be created for + // this call. + MOZ_ASSERT( + stackMapGenerator_.framePushedExcludingOutboundCallArgs.isNothing()); + stackMapGenerator_.framePushedExcludingOutboundCallArgs.emplace( + // However much we've pushed so far + masm.framePushed() + + // Extra space we'll push to get the frame aligned + call->frameAlignAdjustment + + // Extra space we'll push to get the outbound arg area aligned + (stackArgAreaSizeAligned - stackArgAreaSizeUnaligned)); + + call->stackArgAreaSize = stackArgAreaSizeAligned; + + size_t adjustment = call->stackArgAreaSize + call->frameAlignAdjustment; + fr.allocArgArea(adjustment); +} + +ABIArg BaseCompiler::reservePointerArgument(FunctionCall* call) { + return call->abi.next(MIRType::Pointer); +} + +// TODO / OPTIMIZE (Bug 1316821): Note passArg is used only in one place. +// (Or it was, until Luke wandered through, but that can be fixed again.) +// I'm not saying we should manually inline it, but we could hoist the +// dispatch into the caller and have type-specific implementations of +// passArg: passArgI32(), etc. Then those might be inlined, at least in PGO +// builds. +// +// The bulk of the work here (60%) is in the next() call, though. +// +// Notably, since next() is so expensive, StackArgAreaSizeUnaligned() +// becomes expensive too. +// +// Somehow there could be a trick here where the sequence of argument types +// (read from the input stream) leads to a cached entry for +// StackArgAreaSizeUnaligned() and for how to pass arguments... +// +// But at least we could reduce the cost of StackArgAreaSizeUnaligned() by +// first reading the argument types into a (reusable) vector, then we have +// the outgoing size at low cost, and then we can pass args based on the +// info we read. + +void BaseCompiler::passArg(ValType type, const Stk& arg, FunctionCall* call) { + switch (type.kind()) { + case ValType::I32: { + ABIArg argLoc = call->abi.next(MIRType::Int32); + if (argLoc.kind() == ABIArg::Stack) { + ScratchI32 scratch(*this); + loadI32(arg, scratch); + masm.store32(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + } else { + loadI32(arg, RegI32(argLoc.gpr())); + } + break; + } + case ValType::I64: { + ABIArg argLoc = call->abi.next(MIRType::Int64); + if (argLoc.kind() == ABIArg::Stack) { + ScratchI32 scratch(*this); +#ifdef JS_PUNBOX64 + loadI64(arg, fromI32(scratch)); + masm.storePtr(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); +#else + loadI64Low(arg, scratch); + masm.store32(scratch, LowWord(Address(masm.getStackPointer(), + argLoc.offsetFromArgBase()))); + loadI64High(arg, scratch); + masm.store32(scratch, HighWord(Address(masm.getStackPointer(), + argLoc.offsetFromArgBase()))); +#endif + } else { + loadI64(arg, RegI64(argLoc.gpr64())); + } + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + ABIArg argLoc = call->abi.next(MIRType::Simd128); + switch (argLoc.kind()) { + case ABIArg::Stack: { + ScratchV128 scratch(*this); + loadV128(arg, scratch); + masm.storeUnalignedSimd128( + (RegV128)scratch, + Address(masm.getStackPointer(), argLoc.offsetFromArgBase())); + break; + } + case ABIArg::GPR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } + case ABIArg::FPU: { + loadV128(arg, RegV128(argLoc.fpu())); + break; + } +# if defined(JS_CODEGEN_REGISTER_PAIR) + case ABIArg::GPR_PAIR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } +# endif + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::F64: { + ABIArg argLoc = call->abi.next(MIRType::Double); + switch (argLoc.kind()) { + case ABIArg::Stack: { + ScratchF64 scratch(*this); + loadF64(arg, scratch); + masm.storeDouble(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + break; + } +#if defined(JS_CODEGEN_REGISTER_PAIR) + case ABIArg::GPR_PAIR: { +# if defined(JS_CODEGEN_ARM) + ScratchF64 scratch(*this); + loadF64(arg, scratch); + masm.ma_vxfer(scratch, argLoc.evenGpr(), argLoc.oddGpr()); + break; +# else + MOZ_CRASH("BaseCompiler platform hook: passArg F64 pair"); +# endif + } +#endif + case ABIArg::FPU: { + loadF64(arg, RegF64(argLoc.fpu())); + break; + } + case ABIArg::GPR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + break; + } + case ValType::F32: { + ABIArg argLoc = call->abi.next(MIRType::Float32); + switch (argLoc.kind()) { + case ABIArg::Stack: { + ScratchF32 scratch(*this); + loadF32(arg, scratch); + masm.storeFloat32(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + break; + } + case ABIArg::GPR: { + ScratchF32 scratch(*this); + loadF32(arg, scratch); + masm.moveFloat32ToGPR(scratch, argLoc.gpr()); + break; + } + case ABIArg::FPU: { + loadF32(arg, RegF32(argLoc.fpu())); + break; + } +#if defined(JS_CODEGEN_REGISTER_PAIR) + case ABIArg::GPR_PAIR: { + MOZ_CRASH("Unexpected parameter passing discipline"); + } +#endif + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + break; + } + case ValType::Ref: { + ABIArg argLoc = call->abi.next(MIRType::RefOrNull); + if (argLoc.kind() == ABIArg::Stack) { + ScratchRef scratch(*this); + loadRef(arg, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + } else { + loadRef(arg, RegRef(argLoc.gpr())); + } + break; + } + } +} + +CodeOffset BaseCompiler::callDefinition(uint32_t funcIndex, + const FunctionCall& call) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Func); + return masm.call(desc, funcIndex); +} + +CodeOffset BaseCompiler::callSymbolic(SymbolicAddress callee, + const FunctionCall& call) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Symbolic); + return masm.call(desc, callee); +} + +// Precondition: sync() + +class OutOfLineAbortingTrap : public OutOfLineCode { + Trap trap_; + BytecodeOffset off_; + + public: + OutOfLineAbortingTrap(Trap trap, BytecodeOffset off) + : trap_(trap), off_(off) {} + + virtual void generate(MacroAssembler* masm) override { + masm->wasmTrap(trap_, off_); + MOZ_ASSERT(!rejoin()->bound()); + } +}; + +bool BaseCompiler::callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex, + const Stk& indexVal, const FunctionCall& call, + CodeOffset* fastCallOffset, + CodeOffset* slowCallOffset) { + CallIndirectId callIndirectId = + CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex); + MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS); + + const TableDesc& table = moduleEnv_.tables[tableIndex]; + + loadI32(indexVal, RegI32(WasmTableCallIndexReg)); + + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Indirect); + CalleeDesc callee = + CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId); + OutOfLineCode* oob = addOutOfLineCode( + new (alloc_) OutOfLineAbortingTrap(Trap::OutOfBounds, bytecodeOffset())); + if (!oob) { + return false; + } + Label* nullCheckFailed = nullptr; +#ifndef WASM_HAS_HEAPREG + OutOfLineCode* nullref = addOutOfLineCode(new (alloc_) OutOfLineAbortingTrap( + Trap::IndirectCallToNull, bytecodeOffset())); + if (!oob) { + return false; + } + nullCheckFailed = nullref->entry(); +#endif + masm.wasmCallIndirect(desc, callee, oob->entry(), nullCheckFailed, + mozilla::Nothing(), fastCallOffset, slowCallOffset); + return true; +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +void BaseCompiler::callRef(const Stk& calleeRef, const FunctionCall& call, + CodeOffset* fastCallOffset, + CodeOffset* slowCallOffset) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::FuncRef); + CalleeDesc callee = CalleeDesc::wasmFuncRef(); + + loadRef(calleeRef, RegRef(WasmCallRefReg)); + masm.wasmCallRef(desc, callee, fastCallOffset, slowCallOffset); +} +#endif + +// Precondition: sync() + +CodeOffset BaseCompiler::callImport(unsigned instanceDataOffset, + const FunctionCall& call) { + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Import); + CalleeDesc callee = CalleeDesc::import(instanceDataOffset); + return masm.wasmCallImport(desc, callee); +} + +CodeOffset BaseCompiler::builtinCall(SymbolicAddress builtin, + const FunctionCall& call) { + return callSymbolic(builtin, call); +} + +CodeOffset BaseCompiler::builtinInstanceMethodCall( + const SymbolicAddressSignature& builtin, const ABIArg& instanceArg, + const FunctionCall& call) { +#ifndef RABALDR_PIN_INSTANCE + // Builtin method calls assume the instance register has been set. + fr.loadInstancePtr(InstanceReg); +#endif + CallSiteDesc desc(bytecodeOffset(), CallSiteDesc::Symbolic); + return masm.wasmCallBuiltinInstanceMethod(desc, instanceArg, builtin.identity, + builtin.failureMode); +} + +bool BaseCompiler::pushCallResults(const FunctionCall& call, ResultType type, + const StackResultsLoc& loc) { +#if defined(JS_CODEGEN_ARM) + // pushResults currently bypasses special case code in captureReturnedFxx() + // that converts GPR results to FPR results for systemABI+softFP. If we + // ever start using that combination for calls we need more code. This + // assert is stronger than we need - we only care about results in return + // registers - but that's OK. + MOZ_ASSERT(!call.usesSystemAbi || call.hardFP); +#endif + return pushResults(type, fr.stackResultsBase(loc.bytes())); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Exception handling + +// Abstracted helper for throwing, used for throw, rethrow, and rethrowing +// at the end of a series of catch blocks (if none matched the exception). +bool BaseCompiler::throwFrom(RegRef exn) { + pushRef(exn); + + // ThrowException invokes a trap, and the rest is dead code. + return emitInstanceCall(SASigThrowException); +} + +void BaseCompiler::loadTag(RegPtr instance, uint32_t tagIndex, RegRef tagDst) { + size_t offset = + Instance::offsetInData(moduleEnv_.offsetOfTagInstanceData(tagIndex)); + masm.loadPtr(Address(instance, offset), tagDst); +} + +void BaseCompiler::consumePendingException(RegRef* exnDst, RegRef* tagDst) { + RegPtr pendingAddr = RegPtr(PreBarrierReg); + needPtr(pendingAddr); + masm.computeEffectiveAddress( + Address(InstanceReg, Instance::offsetOfPendingException()), pendingAddr); + *exnDst = needRef(); + masm.loadPtr(Address(pendingAddr, 0), *exnDst); + emitBarrieredClear(pendingAddr); + + *tagDst = needRef(); + masm.computeEffectiveAddress( + Address(InstanceReg, Instance::offsetOfPendingExceptionTag()), + pendingAddr); + masm.loadPtr(Address(pendingAddr, 0), *tagDst); + emitBarrieredClear(pendingAddr); + freePtr(pendingAddr); +} + +bool BaseCompiler::startTryNote(size_t* tryNoteIndex) { + // Check the previous try note to ensure that we don't share an edge with + // it that could lead to ambiguity. Insert a nop, if required. + TryNoteVector& tryNotes = masm.tryNotes(); + if (tryNotes.length() > 0) { + const TryNote& previous = tryNotes.back(); + uint32_t currentOffset = masm.currentOffset(); + if (previous.tryBodyBegin() == currentOffset || + previous.tryBodyEnd() == currentOffset) { + masm.nop(); + } + } + + // Mark the beginning of the try note + wasm::TryNote tryNote = wasm::TryNote(); + tryNote.setTryBodyBegin(masm.currentOffset()); + return masm.append(tryNote, tryNoteIndex); +} + +void BaseCompiler::finishTryNote(size_t tryNoteIndex) { + TryNoteVector& tryNotes = masm.tryNotes(); + TryNote& tryNote = tryNotes[tryNoteIndex]; + + // Disallow zero-length try notes by inserting a no-op + if (tryNote.tryBodyBegin() == masm.currentOffset()) { + masm.nop(); + } + + // Check the previous try note to ensure that we don't share an edge with + // it that could lead to ambiguity. Insert a nop, if required. + if (tryNotes.length() > 0) { + const TryNote& previous = tryNotes.back(); + uint32_t currentOffset = masm.currentOffset(); + if (previous.tryBodyEnd() == currentOffset) { + masm.nop(); + } + } + + // Don't set the end of the try note if we've OOM'ed, as the above nop's may + // not have been placed. This is okay as this compilation will be thrown + // away. + if (masm.oom()) { + return; + } + + // Mark the end of the try note + tryNote.setTryBodyEnd(masm.currentOffset()); +} + +//////////////////////////////////////////////////////////// +// +// Platform-specific popping and register targeting. + +// The simple popping methods pop values into targeted registers; the caller +// can free registers using standard functions. These are always called +// popXForY where X says something about types and Y something about the +// operation being targeted. + +RegI32 BaseCompiler::needRotate64Temp() { +#if defined(JS_CODEGEN_X86) + return needI32(); +#elif defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ + defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) + return RegI32::Invalid(); +#else + MOZ_CRASH("BaseCompiler platform hook: needRotate64Temp"); +#endif +} + +void BaseCompiler::popAndAllocateForDivAndRemI32(RegI32* r0, RegI32* r1, + RegI32* reserved) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r0 must be eax, and edx will be clobbered. + need2xI32(specific_.eax, specific_.edx); + *r1 = popI32(); + *r0 = popI32ToSpecific(specific_.eax); + *reserved = specific_.edx; +#else + pop2xI32(r0, r1); +#endif +} + +static void QuotientI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd, + RegI32 reserved, IsUnsigned isUnsigned) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + masm.quotient32(rs, rsd, reserved, isUnsigned); +#else + masm.quotient32(rs, rsd, isUnsigned); +#endif +} + +static void RemainderI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd, + RegI32 reserved, IsUnsigned isUnsigned) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + masm.remainder32(rs, rsd, reserved, isUnsigned); +#else + masm.remainder32(rs, rsd, isUnsigned); +#endif +} + +void BaseCompiler::popAndAllocateForMulI64(RegI64* r0, RegI64* r1, + RegI32* temp) { +#if defined(JS_CODEGEN_X64) + pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_X86) + // lhsDest must be edx:eax and rhs must not be that. + needI64(specific_.edx_eax); + *r1 = popI64(); + *r0 = popI64ToSpecific(specific_.edx_eax); + *temp = needI32(); +#elif defined(JS_CODEGEN_MIPS64) + pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_ARM) + pop2xI64(r0, r1); + *temp = needI32(); +#elif defined(JS_CODEGEN_ARM64) + pop2xI64(r0, r1); +#elif defined(JS_CODEGEN_LOONG64) + pop2xI64(r0, r1); +#else + MOZ_CRASH("BaseCompiler porting interface: popAndAllocateForMulI64"); +#endif +} + +#ifndef RABALDR_INT_DIV_I64_CALLOUT + +void BaseCompiler::popAndAllocateForDivAndRemI64(RegI64* r0, RegI64* r1, + RegI64* reserved, + IsRemainder isRemainder) { +# if defined(JS_CODEGEN_X64) + // r0 must be rax, and rdx will be clobbered. + need2xI64(specific_.rax, specific_.rdx); + *r1 = popI64(); + *r0 = popI64ToSpecific(specific_.rax); + *reserved = specific_.rdx; +# elif defined(JS_CODEGEN_ARM64) + pop2xI64(r0, r1); + if (isRemainder) { + *reserved = needI64(); + } +# else + pop2xI64(r0, r1); +# endif +} + +static void QuotientI64(MacroAssembler& masm, RegI64 rhs, RegI64 srcDest, + RegI64 reserved, IsUnsigned isUnsigned) { +# if defined(JS_CODEGEN_X64) + // The caller must set up the following situation. + MOZ_ASSERT(srcDest.reg == rax); + MOZ_ASSERT(reserved.reg == rdx); + if (isUnsigned) { + masm.xorq(rdx, rdx); + masm.udivq(rhs.reg); + } else { + masm.cqo(); + masm.idivq(rhs.reg); + } +# elif defined(JS_CODEGEN_MIPS64) + MOZ_ASSERT(reserved.isInvalid()); + if (isUnsigned) { + masm.as_ddivu(srcDest.reg, rhs.reg); + } else { + masm.as_ddiv(srcDest.reg, rhs.reg); + } + masm.as_mflo(srcDest.reg); +# elif defined(JS_CODEGEN_ARM64) + MOZ_ASSERT(reserved.isInvalid()); + ARMRegister sd(srcDest.reg, 64); + ARMRegister r(rhs.reg, 64); + if (isUnsigned) { + masm.Udiv(sd, sd, r); + } else { + masm.Sdiv(sd, sd, r); + } +# elif defined(JS_CODEGEN_LOONG64) + if (isUnsigned) { + masm.as_div_du(srcDest.reg, srcDest.reg, rhs.reg); + } else { + masm.as_div_d(srcDest.reg, srcDest.reg, rhs.reg); + } +# else + MOZ_CRASH("BaseCompiler platform hook: quotientI64"); +# endif +} + +static void RemainderI64(MacroAssembler& masm, RegI64 rhs, RegI64 srcDest, + RegI64 reserved, IsUnsigned isUnsigned) { +# if defined(JS_CODEGEN_X64) + // The caller must set up the following situation. + MOZ_ASSERT(srcDest.reg == rax); + MOZ_ASSERT(reserved.reg == rdx); + + if (isUnsigned) { + masm.xorq(rdx, rdx); + masm.udivq(rhs.reg); + } else { + masm.cqo(); + masm.idivq(rhs.reg); + } + masm.movq(rdx, rax); +# elif defined(JS_CODEGEN_MIPS64) + MOZ_ASSERT(reserved.isInvalid()); + if (isUnsigned) { + masm.as_ddivu(srcDest.reg, rhs.reg); + } else { + masm.as_ddiv(srcDest.reg, rhs.reg); + } + masm.as_mfhi(srcDest.reg); +# elif defined(JS_CODEGEN_ARM64) + ARMRegister sd(srcDest.reg, 64); + ARMRegister r(rhs.reg, 64); + ARMRegister t(reserved.reg, 64); + if (isUnsigned) { + masm.Udiv(t, sd, r); + } else { + masm.Sdiv(t, sd, r); + } + masm.Mul(t, t, r); + masm.Sub(sd, sd, t); +# elif defined(JS_CODEGEN_LOONG64) + if (isUnsigned) { + masm.as_mod_du(srcDest.reg, srcDest.reg, rhs.reg); + } else { + masm.as_mod_d(srcDest.reg, srcDest.reg, rhs.reg); + } +# else + MOZ_CRASH("BaseCompiler platform hook: remainderI64"); +# endif +} + +#endif // RABALDR_INT_DIV_I64_CALLOUT + +RegI32 BaseCompiler::popI32RhsForShift() { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r1 must be ecx for a variable shift, unless BMI2 is available. + if (!Assembler::HasBMI2()) { + return popI32(specific_.ecx); + } +#endif + RegI32 r = popI32(); +#if defined(JS_CODEGEN_ARM) + masm.and32(Imm32(31), r); +#endif + return r; +} + +RegI32 BaseCompiler::popI32RhsForShiftI64() { +#if defined(JS_CODEGEN_X86) + // A limitation in the x86 masm requires ecx here + return popI32(specific_.ecx); +#elif defined(JS_CODEGEN_X64) + if (!Assembler::HasBMI2()) { + return popI32(specific_.ecx); + } + return popI32(); +#else + return popI32(); +#endif +} + +RegI64 BaseCompiler::popI64RhsForShift() { +#if defined(JS_CODEGEN_X86) + // r1 must be ecx for a variable shift. + needI32(specific_.ecx); + return popI64ToSpecific(widenI32(specific_.ecx)); +#else +# if defined(JS_CODEGEN_X64) + // r1 must be rcx for a variable shift, unless BMI2 is available. + if (!Assembler::HasBMI2()) { + needI64(specific_.rcx); + return popI64ToSpecific(specific_.rcx); + } +# endif + // No masking is necessary on 64-bit platforms, and on arm32 the masm + // implementation masks. + return popI64(); +#endif +} + +RegI32 BaseCompiler::popI32RhsForRotate() { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r1 must be ecx for a variable rotate. + return popI32(specific_.ecx); +#else + return popI32(); +#endif +} + +RegI64 BaseCompiler::popI64RhsForRotate() { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + // r1 must be ecx for a variable rotate. + needI32(specific_.ecx); + return popI64ToSpecific(widenI32(specific_.ecx)); +#else + return popI64(); +#endif +} + +void BaseCompiler::popI32ForSignExtendI64(RegI64* r0) { +#if defined(JS_CODEGEN_X86) + // r0 must be edx:eax for cdq + need2xI32(specific_.edx, specific_.eax); + *r0 = specific_.edx_eax; + popI32ToSpecific(specific_.eax); +#else + *r0 = widenI32(popI32()); +#endif +} + +void BaseCompiler::popI64ForSignExtendI64(RegI64* r0) { +#if defined(JS_CODEGEN_X86) + // r0 must be edx:eax for cdq + need2xI32(specific_.edx, specific_.eax); + // Low on top, high underneath + *r0 = popI64ToSpecific(specific_.edx_eax); +#else + *r0 = popI64(); +#endif +} + +class OutOfLineTruncateCheckF32OrF64ToI32 : public OutOfLineCode { + AnyReg src; + RegI32 dest; + TruncFlags flags; + BytecodeOffset off; + + public: + OutOfLineTruncateCheckF32OrF64ToI32(AnyReg src, RegI32 dest, TruncFlags flags, + BytecodeOffset off) + : src(src), dest(dest), flags(flags), off(off) {} + + virtual void generate(MacroAssembler* masm) override { + if (src.tag == AnyReg::F32) { + masm->oolWasmTruncateCheckF32ToI32(src.f32(), dest, flags, off, rejoin()); + } else if (src.tag == AnyReg::F64) { + masm->oolWasmTruncateCheckF64ToI32(src.f64(), dest, flags, off, rejoin()); + } else { + MOZ_CRASH("unexpected type"); + } + } +}; + +bool BaseCompiler::truncateF32ToI32(RegF32 src, RegI32 dest, TruncFlags flags) { + BytecodeOffset off = bytecodeOffset(); + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI32( + AnyReg(src), dest, flags, off)); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateFloat32ToUInt32(src, dest, isSaturating, ool->entry()); + } else { + masm.wasmTruncateFloat32ToInt32(src, dest, isSaturating, ool->entry()); + } + masm.bind(ool->rejoin()); + return true; +} + +bool BaseCompiler::truncateF64ToI32(RegF64 src, RegI32 dest, TruncFlags flags) { + BytecodeOffset off = bytecodeOffset(); + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI32( + AnyReg(src), dest, flags, off)); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateDoubleToUInt32(src, dest, isSaturating, ool->entry()); + } else { + masm.wasmTruncateDoubleToInt32(src, dest, isSaturating, ool->entry()); + } + masm.bind(ool->rejoin()); + return true; +} + +class OutOfLineTruncateCheckF32OrF64ToI64 : public OutOfLineCode { + AnyReg src; + RegI64 dest; + TruncFlags flags; + BytecodeOffset off; + + public: + OutOfLineTruncateCheckF32OrF64ToI64(AnyReg src, RegI64 dest, TruncFlags flags, + BytecodeOffset off) + : src(src), dest(dest), flags(flags), off(off) {} + + virtual void generate(MacroAssembler* masm) override { + if (src.tag == AnyReg::F32) { + masm->oolWasmTruncateCheckF32ToI64(src.f32(), dest, flags, off, rejoin()); + } else if (src.tag == AnyReg::F64) { + masm->oolWasmTruncateCheckF64ToI64(src.f64(), dest, flags, off, rejoin()); + } else { + MOZ_CRASH("unexpected type"); + } + } +}; + +#ifndef RABALDR_FLOAT_TO_I64_CALLOUT + +RegF64 BaseCompiler::needTempForFloatingToI64(TruncFlags flags) { +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + if (flags & TRUNC_UNSIGNED) { + return needF64(); + } +# endif + return RegF64::Invalid(); +} + +bool BaseCompiler::truncateF32ToI64(RegF32 src, RegI64 dest, TruncFlags flags, + RegF64 temp) { + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64( + AnyReg(src), dest, flags, bytecodeOffset())); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateFloat32ToUInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } else { + masm.wasmTruncateFloat32ToInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } + return true; +} + +bool BaseCompiler::truncateF64ToI64(RegF64 src, RegI64 dest, TruncFlags flags, + RegF64 temp) { + OutOfLineCode* ool = + addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64( + AnyReg(src), dest, flags, bytecodeOffset())); + if (!ool) { + return false; + } + bool isSaturating = flags & TRUNC_SATURATING; + if (flags & TRUNC_UNSIGNED) { + masm.wasmTruncateDoubleToUInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } else { + masm.wasmTruncateDoubleToInt64(src, dest, isSaturating, ool->entry(), + ool->rejoin(), temp); + } + return true; +} + +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT + +RegI32 BaseCompiler::needConvertI64ToFloatTemp(ValType to, bool isUnsigned) { + bool needs = false; + if (to == ValType::F64) { + needs = isUnsigned && masm.convertUInt64ToDoubleNeedsTemp(); + } else { +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + needs = true; +# endif + } + return needs ? needI32() : RegI32::Invalid(); +} + +void BaseCompiler::convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest, + RegI32 temp) { + if (isUnsigned) { + masm.convertUInt64ToFloat32(src, dest, temp); + } else { + masm.convertInt64ToFloat32(src, dest); + } +} + +void BaseCompiler::convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest, + RegI32 temp) { + if (isUnsigned) { + masm.convertUInt64ToDouble(src, dest, temp); + } else { + masm.convertInt64ToDouble(src, dest); + } +} + +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + +////////////////////////////////////////////////////////////////////// +// +// Global variable access. + +Address BaseCompiler::addressOfGlobalVar(const GlobalDesc& global, RegPtr tmp) { + uint32_t globalToInstanceOffset = Instance::offsetInData(global.offset()); +#ifdef RABALDR_PIN_INSTANCE + movePtr(RegPtr(InstanceReg), tmp); +#else + fr.loadInstancePtr(tmp); +#endif + if (global.isIndirect()) { + masm.loadPtr(Address(tmp, globalToInstanceOffset), tmp); + return Address(tmp, 0); + } + return Address(tmp, globalToInstanceOffset); +} + +////////////////////////////////////////////////////////////////////// +// +// Table access. + +Address BaseCompiler::addressOfTableField(uint32_t tableIndex, + uint32_t fieldOffset, + RegPtr instance) { + uint32_t tableToInstanceOffset = wasm::Instance::offsetInData( + moduleEnv_.offsetOfTableInstanceData(tableIndex) + fieldOffset); + return Address(instance, tableToInstanceOffset); +} + +void BaseCompiler::loadTableLength(uint32_t tableIndex, RegPtr instance, + RegI32 length) { + masm.load32(addressOfTableField( + tableIndex, offsetof(TableInstanceData, length), instance), + length); +} + +void BaseCompiler::loadTableElements(uint32_t tableIndex, RegPtr instance, + RegPtr elements) { + masm.loadPtr(addressOfTableField( + tableIndex, offsetof(TableInstanceData, elements), instance), + elements); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Basic emitters for simple operators. + +static void AddI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.add32(rs, rsd); +} + +static void AddImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.add32(Imm32(c), rsd); +} + +static void SubI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.sub32(rs, rsd); +} + +static void SubImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.sub32(Imm32(c), rsd); +} + +static void MulI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.mul32(rs, rsd); +} + +static void OrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.or32(rs, rsd); +} + +static void OrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.or32(Imm32(c), rsd); +} + +static void AndI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.and32(rs, rsd); +} + +static void AndImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.and32(Imm32(c), rsd); +} + +static void XorI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.xor32(rs, rsd); +} + +static void XorImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.xor32(Imm32(c), rsd); +} + +static void ClzI32(MacroAssembler& masm, RegI32 rsd) { + masm.clz32(rsd, rsd, IsKnownNotZero(false)); +} + +static void CtzI32(MacroAssembler& masm, RegI32 rsd) { + masm.ctz32(rsd, rsd, IsKnownNotZero(false)); +} + +// Currently common to PopcntI32 and PopcntI64 +static RegI32 PopcntTemp(BaseCompiler& bc) { +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + return AssemblerX86Shared::HasPOPCNT() ? RegI32::Invalid() : bc.needI32(); +#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + return bc.needI32(); +#else + MOZ_CRASH("BaseCompiler platform hook: PopcntTemp"); +#endif +} + +static void PopcntI32(BaseCompiler& bc, RegI32 rsd, RegI32 temp) { + bc.masm.popcnt32(rsd, rsd, temp); +} + +static void ShlI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.lshift32(rs, rsd); +} + +static void ShlImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.lshift32(Imm32(c & 31), rsd); +} + +static void ShrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rshift32Arithmetic(rs, rsd); +} + +static void ShrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rshift32Arithmetic(Imm32(c & 31), rsd); +} + +static void ShrUI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rshift32(rs, rsd); +} + +static void ShrUImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rshift32(Imm32(c & 31), rsd); +} + +static void RotlI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rotateLeft(rs, rsd, rsd); +} + +static void RotlImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rotateLeft(Imm32(c & 31), rsd, rsd); +} + +static void RotrI32(MacroAssembler& masm, RegI32 rs, RegI32 rsd) { + masm.rotateRight(rs, rsd, rsd); +} + +static void RotrImmI32(MacroAssembler& masm, int32_t c, RegI32 rsd) { + masm.rotateRight(Imm32(c & 31), rsd, rsd); +} + +static void EqzI32(MacroAssembler& masm, RegI32 rsd) { + masm.cmp32Set(Assembler::Equal, rsd, Imm32(0), rsd); +} + +static void WrapI64ToI32(MacroAssembler& masm, RegI64 rs, RegI32 rd) { + masm.move64To32(rs, rd); +} + +static void AddI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.add64(rs, rsd); +} + +static void AddImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.add64(Imm64(c), rsd); +} + +static void SubI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.sub64(rs, rsd); +} + +static void SubImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.sub64(Imm64(c), rsd); +} + +static void OrI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.or64(rs, rsd); +} + +static void OrImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.or64(Imm64(c), rsd); +} + +static void AndI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.and64(rs, rsd); +} + +static void AndImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.and64(Imm64(c), rsd); +} + +static void XorI64(MacroAssembler& masm, RegI64 rs, RegI64 rsd) { + masm.xor64(rs, rsd); +} + +static void XorImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.xor64(Imm64(c), rsd); +} + +static void ClzI64(BaseCompiler& bc, RegI64 rsd) { + bc.masm.clz64(rsd, bc.lowPart(rsd)); + bc.maybeClearHighPart(rsd); +} + +static void CtzI64(BaseCompiler& bc, RegI64 rsd) { + bc.masm.ctz64(rsd, bc.lowPart(rsd)); + bc.maybeClearHighPart(rsd); +} + +static void PopcntI64(BaseCompiler& bc, RegI64 rsd, RegI32 temp) { + bc.masm.popcnt64(rsd, rsd, temp); +} + +static void ShlI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) { + bc.masm.lshift64(bc.lowPart(rs), rsd); +} + +static void ShlImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.lshift64(Imm32(c & 63), rsd); +} + +static void ShrI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) { + bc.masm.rshift64Arithmetic(bc.lowPart(rs), rsd); +} + +static void ShrImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.rshift64Arithmetic(Imm32(c & 63), rsd); +} + +static void ShrUI64(BaseCompiler& bc, RegI64 rs, RegI64 rsd) { + bc.masm.rshift64(bc.lowPart(rs), rsd); +} + +static void ShrUImmI64(MacroAssembler& masm, int64_t c, RegI64 rsd) { + masm.rshift64(Imm32(c & 63), rsd); +} + +static void EqzI64(MacroAssembler& masm, RegI64 rs, RegI32 rd) { +#ifdef JS_PUNBOX64 + masm.cmpPtrSet(Assembler::Equal, rs.reg, ImmWord(0), rd); +#else + MOZ_ASSERT(rs.low == rd); + masm.or32(rs.high, rs.low); + masm.cmp32Set(Assembler::Equal, rs.low, Imm32(0), rd); +#endif +} + +static void AddF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.addDouble(rs, rsd); +} + +static void SubF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.subDouble(rs, rsd); +} + +static void MulF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.mulDouble(rs, rsd); +} + +static void DivF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd) { + masm.divDouble(rs, rsd); +} + +static void MinF64(BaseCompiler& bc, RegF64 rs, RegF64 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): see comment in MinF32. +#ifdef RABALDR_SCRATCH_F64 + ScratchF64 zero(bc.ra); +#else + ScratchF64 zero(bc.masm); +#endif + bc.masm.loadConstantDouble(0, zero); + bc.masm.subDouble(zero, rsd); + bc.masm.subDouble(zero, rs); + bc.masm.minDouble(rs, rsd, HandleNaNSpecially(true)); +} + +static void MaxF64(BaseCompiler& bc, RegF64 rs, RegF64 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): see comment in MinF32. +#ifdef RABALDR_SCRATCH_F64 + ScratchF64 zero(bc.ra); +#else + ScratchF64 zero(bc.masm); +#endif + bc.masm.loadConstantDouble(0, zero); + bc.masm.subDouble(zero, rsd); + bc.masm.subDouble(zero, rs); + bc.masm.maxDouble(rs, rsd, HandleNaNSpecially(true)); +} + +static void CopysignF64(MacroAssembler& masm, RegF64 rs, RegF64 rsd, + RegI64 temp0, RegI64 temp1) { + masm.moveDoubleToGPR64(rsd, temp0); + masm.moveDoubleToGPR64(rs, temp1); + masm.and64(Imm64(INT64_MAX), temp0); + masm.and64(Imm64(INT64_MIN), temp1); + masm.or64(temp1, temp0); + masm.moveGPR64ToDouble(temp0, rsd); +} + +static void AbsF64(MacroAssembler& masm, RegF64 rsd) { + masm.absDouble(rsd, rsd); +} + +static void NegateF64(MacroAssembler& masm, RegF64 rsd) { + masm.negateDouble(rsd); +} + +static void SqrtF64(MacroAssembler& masm, RegF64 rsd) { + masm.sqrtDouble(rsd, rsd); +} + +static void AddF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.addFloat32(rs, rsd); +} + +static void SubF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.subFloat32(rs, rsd); +} + +static void MulF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.mulFloat32(rs, rsd); +} + +static void DivF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd) { + masm.divFloat32(rs, rsd); +} + +static void MinF32(BaseCompiler& bc, RegF32 rs, RegF32 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): Don't do this if one of the operands + // is known to be a constant. +#ifdef RABALDR_SCRATCH_F32 + ScratchF32 zero(bc.ra); +#else + ScratchF32 zero(bc.masm); +#endif + bc.masm.loadConstantFloat32(0.f, zero); + bc.masm.subFloat32(zero, rsd); + bc.masm.subFloat32(zero, rs); + bc.masm.minFloat32(rs, rsd, HandleNaNSpecially(true)); +} + +static void MaxF32(BaseCompiler& bc, RegF32 rs, RegF32 rsd) { + // Convert signaling NaN to quiet NaNs. + // + // TODO / OPTIMIZE (bug 1316824): see comment in MinF32. +#ifdef RABALDR_SCRATCH_F32 + ScratchF32 zero(bc.ra); +#else + ScratchF32 zero(bc.masm); +#endif + bc.masm.loadConstantFloat32(0.f, zero); + bc.masm.subFloat32(zero, rsd); + bc.masm.subFloat32(zero, rs); + bc.masm.maxFloat32(rs, rsd, HandleNaNSpecially(true)); +} + +static void CopysignF32(MacroAssembler& masm, RegF32 rs, RegF32 rsd, + RegI32 temp0, RegI32 temp1) { + masm.moveFloat32ToGPR(rsd, temp0); + masm.moveFloat32ToGPR(rs, temp1); + masm.and32(Imm32(INT32_MAX), temp0); + masm.and32(Imm32(INT32_MIN), temp1); + masm.or32(temp1, temp0); + masm.moveGPRToFloat32(temp0, rsd); +} + +static void AbsF32(MacroAssembler& masm, RegF32 rsd) { + masm.absFloat32(rsd, rsd); +} + +static void NegateF32(MacroAssembler& masm, RegF32 rsd) { + masm.negateFloat(rsd); +} + +static void SqrtF32(MacroAssembler& masm, RegF32 rsd) { + masm.sqrtFloat32(rsd, rsd); +} + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT +static void ConvertI64ToF32(MacroAssembler& masm, RegI64 rs, RegF32 rd) { + masm.convertInt64ToFloat32(rs, rd); +} + +static void ConvertI64ToF64(MacroAssembler& masm, RegI64 rs, RegF64 rd) { + masm.convertInt64ToDouble(rs, rd); +} +#endif + +static void ReinterpretF32AsI32(MacroAssembler& masm, RegF32 rs, RegI32 rd) { + masm.moveFloat32ToGPR(rs, rd); +} + +static void ReinterpretF64AsI64(MacroAssembler& masm, RegF64 rs, RegI64 rd) { + masm.moveDoubleToGPR64(rs, rd); +} + +static void ConvertF64ToF32(MacroAssembler& masm, RegF64 rs, RegF32 rd) { + masm.convertDoubleToFloat32(rs, rd); +} + +static void ConvertI32ToF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) { + masm.convertInt32ToFloat32(rs, rd); +} + +static void ConvertU32ToF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) { + masm.convertUInt32ToFloat32(rs, rd); +} + +static void ConvertF32ToF64(MacroAssembler& masm, RegF32 rs, RegF64 rd) { + masm.convertFloat32ToDouble(rs, rd); +} + +static void ConvertI32ToF64(MacroAssembler& masm, RegI32 rs, RegF64 rd) { + masm.convertInt32ToDouble(rs, rd); +} + +static void ConvertU32ToF64(MacroAssembler& masm, RegI32 rs, RegF64 rd) { + masm.convertUInt32ToDouble(rs, rd); +} + +static void ReinterpretI32AsF32(MacroAssembler& masm, RegI32 rs, RegF32 rd) { + masm.moveGPRToFloat32(rs, rd); +} + +static void ReinterpretI64AsF64(MacroAssembler& masm, RegI64 rs, RegF64 rd) { + masm.moveGPR64ToDouble(rs, rd); +} + +static void ExtendI32_8(BaseCompiler& bc, RegI32 rsd) { +#ifdef JS_CODEGEN_X86 + if (!bc.ra.isSingleByteI32(rsd)) { + ScratchI8 scratch(bc.ra); + bc.masm.move32(rsd, scratch); + bc.masm.move8SignExtend(scratch, rsd); + return; + } +#endif + bc.masm.move8SignExtend(rsd, rsd); +} + +static void ExtendI32_16(MacroAssembler& masm, RegI32 rsd) { + masm.move16SignExtend(rsd, rsd); +} + +void BaseCompiler::emitMultiplyI64() { + RegI64 r, rs; + RegI32 temp; + popAndAllocateForMulI64(&r, &rs, &temp); + masm.mul64(rs, r, temp); + maybeFree(temp); + freeI64(rs); + pushI64(r); +} + +template +void BaseCompiler::quotientOrRemainder( + RegType rs, RegType rsd, RegType reserved, IsUnsigned isUnsigned, + ZeroOnOverflow zeroOnOverflow, bool isConst, IntType c, + void (*operate)(MacroAssembler& masm, RegType rs, RegType rsd, + RegType reserved, IsUnsigned isUnsigned)) { + Label done; + if (!isConst || c == 0) { + checkDivideByZero(rs); + } + if (!isUnsigned && (!isConst || c == -1)) { + checkDivideSignedOverflow(rs, rsd, &done, zeroOnOverflow); + } + operate(masm, rs, rsd, reserved, isUnsigned); + masm.bind(&done); +} + +void BaseCompiler::emitQuotientI32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI32 r = popI32(); + Label positive; + masm.branchTest32(Assembler::NotSigned, r, r, &positive); + masm.add32(Imm32(c - 1), r); + masm.bind(&positive); + + masm.rshift32Arithmetic(Imm32(power & 31), r); + pushI32(r); + } + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(false), isConst, c, QuotientI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitQuotientU32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI32 r = popI32(); + masm.rshift32(Imm32(power & 31), r); + pushI32(r); + } + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), + ZeroOnOverflow(false), isConst, c, QuotientI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitRemainderI32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI32 r = popI32(); + RegI32 temp = needI32(); + moveI32(r, temp); + + Label positive; + masm.branchTest32(Assembler::NotSigned, temp, temp, &positive); + masm.add32(Imm32(c - 1), temp); + masm.bind(&positive); + + masm.rshift32Arithmetic(Imm32(power & 31), temp); + masm.lshift32(Imm32(power & 31), temp); + masm.sub32(temp, r); + freeI32(temp); + + pushI32(r); + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(true), isConst, c, RemainderI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitRemainderU32() { + int32_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI32 r = popI32(); + masm.and32(Imm32(c - 1), r); + pushI32(r); + } else { + bool isConst = peekConst(&c); + RegI32 r, rs, reserved; + popAndAllocateForDivAndRemI32(&r, &rs, &reserved); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), ZeroOnOverflow(true), + isConst, c, RemainderI32); + maybeFree(reserved); + freeI32(rs); + pushI32(r); + } +} + +#ifndef RABALDR_INT_DIV_I64_CALLOUT +void BaseCompiler::emitQuotientI64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI64 r = popI64(); + Label positive; + masm.branchTest64(Assembler::NotSigned, r, r, RegI32::Invalid(), + &positive); + masm.add64(Imm64(c - 1), r); + masm.bind(&positive); + + masm.rshift64Arithmetic(Imm32(power & 63), r); + pushI64(r); + } + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(false)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(false), isConst, c, QuotientI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitQuotientU64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 0)) { + if (power != 0) { + RegI64 r = popI64(); + masm.rshift64(Imm32(power & 63), r); + pushI64(r); + } + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(false)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), + ZeroOnOverflow(false), isConst, c, QuotientI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitRemainderI64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI64 r = popI64(); + RegI64 temp = needI64(); + moveI64(r, temp); + + Label positive; + masm.branchTest64(Assembler::NotSigned, temp, temp, RegI32::Invalid(), + &positive); + masm.add64(Imm64(c - 1), temp); + masm.bind(&positive); + + masm.rshift64Arithmetic(Imm32(power & 63), temp); + masm.lshift64(Imm32(power & 63), temp); + masm.sub64(temp, r); + freeI64(temp); + + pushI64(r); + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(true)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(false), + ZeroOnOverflow(true), isConst, c, RemainderI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitRemainderU64() { + int64_t c; + uint_fast8_t power; + if (popConstPositivePowerOfTwo(&c, &power, 1)) { + RegI64 r = popI64(); + masm.and64(Imm64(c - 1), r); + pushI64(r); + } else { + bool isConst = peekConst(&c); + RegI64 r, rs, reserved; + popAndAllocateForDivAndRemI64(&r, &rs, &reserved, IsRemainder(true)); + quotientOrRemainder(rs, r, reserved, IsUnsigned(true), ZeroOnOverflow(true), + isConst, c, RemainderI64); + maybeFree(reserved); + freeI64(rs); + pushI64(r); + } +} +#endif // RABALDR_INT_DIV_I64_CALLOUT + +void BaseCompiler::emitRotrI64() { + int64_t c; + if (popConst(&c)) { + RegI64 r = popI64(); + RegI32 temp = needRotate64Temp(); + masm.rotateRight64(Imm32(c & 63), r, r, temp); + maybeFree(temp); + pushI64(r); + } else { + RegI64 rs = popI64RhsForRotate(); + RegI64 r = popI64(); + masm.rotateRight64(lowPart(rs), r, r, maybeHighPart(rs)); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitRotlI64() { + int64_t c; + if (popConst(&c)) { + RegI64 r = popI64(); + RegI32 temp = needRotate64Temp(); + masm.rotateLeft64(Imm32(c & 63), r, r, temp); + maybeFree(temp); + pushI64(r); + } else { + RegI64 rs = popI64RhsForRotate(); + RegI64 r = popI64(); + masm.rotateLeft64(lowPart(rs), r, r, maybeHighPart(rs)); + freeI64(rs); + pushI64(r); + } +} + +void BaseCompiler::emitEqzI32() { + if (sniffConditionalControlEqz(ValType::I32)) { + return; + } + emitUnop(EqzI32); +} + +void BaseCompiler::emitEqzI64() { + if (sniffConditionalControlEqz(ValType::I64)) { + return; + } + emitUnop(EqzI64); +} + +template +bool BaseCompiler::emitTruncateF32ToI32() { + RegF32 rs = popF32(); + RegI32 rd = needI32(); + if (!truncateF32ToI32(rs, rd, flags)) { + return false; + } + freeF32(rs); + pushI32(rd); + return true; +} + +template +bool BaseCompiler::emitTruncateF64ToI32() { + RegF64 rs = popF64(); + RegI32 rd = needI32(); + if (!truncateF64ToI32(rs, rd, flags)) { + return false; + } + freeF64(rs); + pushI32(rd); + return true; +} + +#ifndef RABALDR_FLOAT_TO_I64_CALLOUT +template +bool BaseCompiler::emitTruncateF32ToI64() { + RegF32 rs = popF32(); + RegI64 rd = needI64(); + RegF64 temp = needTempForFloatingToI64(flags); + if (!truncateF32ToI64(rs, rd, flags, temp)) { + return false; + } + maybeFree(temp); + freeF32(rs); + pushI64(rd); + return true; +} + +template +bool BaseCompiler::emitTruncateF64ToI64() { + RegF64 rs = popF64(); + RegI64 rd = needI64(); + RegF64 temp = needTempForFloatingToI64(flags); + if (!truncateF64ToI64(rs, rd, flags, temp)) { + return false; + } + maybeFree(temp); + freeF64(rs); + pushI64(rd); + return true; +} +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +void BaseCompiler::emitExtendI64_8() { + RegI64 r; + popI64ForSignExtendI64(&r); + masm.move8To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendI64_16() { + RegI64 r; + popI64ForSignExtendI64(&r); + masm.move16To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendI64_32() { + RegI64 r; + popI64ForSignExtendI64(&r); + masm.move32To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendI32ToI64() { + RegI64 r; + popI32ForSignExtendI64(&r); + masm.move32To64SignExtend(lowPart(r), r); + pushI64(r); +} + +void BaseCompiler::emitExtendU32ToI64() { + RegI32 rs = popI32(); + RegI64 rd = widenI32(rs); + masm.move32To64ZeroExtend(rs, rd); + pushI64(rd); +} + +#ifndef RABALDR_I64_TO_FLOAT_CALLOUT +void BaseCompiler::emitConvertU64ToF32() { + RegI64 rs = popI64(); + RegF32 rd = needF32(); + RegI32 temp = needConvertI64ToFloatTemp(ValType::F32, IsUnsigned(true)); + convertI64ToF32(rs, IsUnsigned(true), rd, temp); + maybeFree(temp); + freeI64(rs); + pushF32(rd); +} + +void BaseCompiler::emitConvertU64ToF64() { + RegI64 rs = popI64(); + RegF64 rd = needF64(); + RegI32 temp = needConvertI64ToFloatTemp(ValType::F64, IsUnsigned(true)); + convertI64ToF64(rs, IsUnsigned(true), rd, temp); + maybeFree(temp); + freeI64(rs); + pushF64(rd); +} +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + +//////////////////////////////////////////////////////////// +// +// Machinery for optimized conditional branches. +// +// To disable this optimization it is enough always to return false from +// sniffConditionalControl{Cmp,Eqz}. + +struct BranchState { + union { + struct { + RegI32 lhs; + RegI32 rhs; + int32_t imm; + bool rhsImm; + } i32; + struct { + RegI64 lhs; + RegI64 rhs; + int64_t imm; + bool rhsImm; + } i64; + struct { + RegF32 lhs; + RegF32 rhs; + } f32; + struct { + RegF64 lhs; + RegF64 rhs; + } f64; + }; + + Label* const label; // The target of the branch, never NULL + const StackHeight stackHeight; // The stack base above which to place + // stack-spilled block results, if + // hasBlockResults(). + const bool invertBranch; // If true, invert the sense of the branch + const ResultType resultType; // The result propagated along the edges + + explicit BranchState(Label* label) + : label(label), + stackHeight(StackHeight::Invalid()), + invertBranch(false), + resultType(ResultType::Empty()) {} + + BranchState(Label* label, bool invertBranch) + : label(label), + stackHeight(StackHeight::Invalid()), + invertBranch(invertBranch), + resultType(ResultType::Empty()) {} + + BranchState(Label* label, StackHeight stackHeight, bool invertBranch, + ResultType resultType) + : label(label), + stackHeight(stackHeight), + invertBranch(invertBranch), + resultType(resultType) {} + + bool hasBlockResults() const { return stackHeight.isValid(); } +}; + +void BaseCompiler::setLatentCompare(Assembler::Condition compareOp, + ValType operandType) { + latentOp_ = LatentOp::Compare; + latentType_ = operandType; + latentIntCmp_ = compareOp; +} + +void BaseCompiler::setLatentCompare(Assembler::DoubleCondition compareOp, + ValType operandType) { + latentOp_ = LatentOp::Compare; + latentType_ = operandType; + latentDoubleCmp_ = compareOp; +} + +void BaseCompiler::setLatentEqz(ValType operandType) { + latentOp_ = LatentOp::Eqz; + latentType_ = operandType; +} + +bool BaseCompiler::hasLatentOp() const { return latentOp_ != LatentOp::None; } + +void BaseCompiler::resetLatentOp() { latentOp_ = LatentOp::None; } + +// Emit a conditional branch that optionally and optimally cleans up the CPU +// stack before we branch. +// +// Cond is either Assembler::Condition or Assembler::DoubleCondition. +// +// Lhs is RegI32, RegI64, or RegF32, RegF64, or RegRef. +// +// Rhs is either the same as Lhs, or an immediate expression compatible with +// Lhs "when applicable". + +template +bool BaseCompiler::jumpConditionalWithResults(BranchState* b, Cond cond, + Lhs lhs, Rhs rhs) { + if (b->hasBlockResults()) { + StackHeight resultsBase(0); + if (!topBranchParams(b->resultType, &resultsBase)) { + return false; + } + if (b->stackHeight != resultsBase) { + Label notTaken; + branchTo(b->invertBranch ? cond : Assembler::InvertCondition(cond), lhs, + rhs, ¬Taken); + + // Shuffle stack args. + shuffleStackResultsBeforeBranch(resultsBase, b->stackHeight, + b->resultType); + masm.jump(b->label); + masm.bind(¬Taken); + return true; + } + } + + branchTo(b->invertBranch ? Assembler::InvertCondition(cond) : cond, lhs, rhs, + b->label); + return true; +} + +#ifdef ENABLE_WASM_GC +bool BaseCompiler::jumpConditionalWithResults(BranchState* b, RegRef object, + RefType sourceType, + RefType destType, + bool onSuccess) { + if (b->hasBlockResults()) { + StackHeight resultsBase(0); + if (!topBranchParams(b->resultType, &resultsBase)) { + return false; + } + if (b->stackHeight != resultsBase) { + Label notTaken; + // Temporarily take the result registers so that branchGcHeapType doesn't + // use them. + needIntegerResultRegisters(b->resultType); + branchGcRefType(object, sourceType, destType, ¬Taken, + /*onSuccess=*/b->invertBranch ? !onSuccess : onSuccess); + freeIntegerResultRegisters(b->resultType); + + // Shuffle stack args. + shuffleStackResultsBeforeBranch(resultsBase, b->stackHeight, + b->resultType); + masm.jump(b->label); + masm.bind(¬Taken); + return true; + } + } + + branchGcRefType(object, sourceType, destType, b->label, + /*onSuccess=*/b->invertBranch ? !onSuccess : onSuccess); + return true; +} +#endif + +// sniffConditionalControl{Cmp,Eqz} may modify the latentWhatever_ state in +// the BaseCompiler so that a subsequent conditional branch can be compiled +// optimally. emitBranchSetup() and emitBranchPerform() will consume that +// state. If the latter methods are not called because deadCode_ is true +// then the compiler MUST instead call resetLatentOp() to reset the state. + +template +bool BaseCompiler::sniffConditionalControlCmp(Cond compareOp, + ValType operandType) { + MOZ_ASSERT(latentOp_ == LatentOp::None, + "Latent comparison state not properly reset"); + +#ifdef JS_CODEGEN_X86 + // On x86, latent i64 binary comparisons use too many registers: the + // reserved join register and the lhs and rhs operands require six, but we + // only have five. + if (operandType == ValType::I64) { + return false; + } +#endif + + // No optimization for pointer compares yet. + if (operandType.isRefRepr()) { + return false; + } + + OpBytes op{}; + iter_.peekOp(&op); + switch (op.b0) { + case uint16_t(Op::BrIf): + case uint16_t(Op::If): + case uint16_t(Op::SelectNumeric): + case uint16_t(Op::SelectTyped): + setLatentCompare(compareOp, operandType); + return true; + default: + return false; + } +} + +bool BaseCompiler::sniffConditionalControlEqz(ValType operandType) { + MOZ_ASSERT(latentOp_ == LatentOp::None, + "Latent comparison state not properly reset"); + + OpBytes op{}; + iter_.peekOp(&op); + switch (op.b0) { + case uint16_t(Op::BrIf): + case uint16_t(Op::SelectNumeric): + case uint16_t(Op::SelectTyped): + case uint16_t(Op::If): + setLatentEqz(operandType); + return true; + default: + return false; + } +} + +void BaseCompiler::emitBranchSetup(BranchState* b) { + // Avoid allocating operands to latentOp_ to result registers. + if (b->hasBlockResults()) { + needResultRegisters(b->resultType); + } + + // Set up fields so that emitBranchPerform() need not switch on latentOp_. + switch (latentOp_) { + case LatentOp::None: { + latentIntCmp_ = Assembler::NotEqual; + latentType_ = ValType::I32; + b->i32.lhs = popI32(); + b->i32.rhsImm = true; + b->i32.imm = 0; + break; + } + case LatentOp::Compare: { + switch (latentType_.kind()) { + case ValType::I32: { + if (popConst(&b->i32.imm)) { + b->i32.lhs = popI32(); + b->i32.rhsImm = true; + } else { + pop2xI32(&b->i32.lhs, &b->i32.rhs); + b->i32.rhsImm = false; + } + break; + } + case ValType::I64: { + pop2xI64(&b->i64.lhs, &b->i64.rhs); + b->i64.rhsImm = false; + break; + } + case ValType::F32: { + pop2xF32(&b->f32.lhs, &b->f32.rhs); + break; + } + case ValType::F64: { + pop2xF64(&b->f64.lhs, &b->f64.rhs); + break; + } + default: { + MOZ_CRASH("Unexpected type for LatentOp::Compare"); + } + } + break; + } + case LatentOp::Eqz: { + switch (latentType_.kind()) { + case ValType::I32: { + latentIntCmp_ = Assembler::Equal; + b->i32.lhs = popI32(); + b->i32.rhsImm = true; + b->i32.imm = 0; + break; + } + case ValType::I64: { + latentIntCmp_ = Assembler::Equal; + b->i64.lhs = popI64(); + b->i64.rhsImm = true; + b->i64.imm = 0; + break; + } + default: { + MOZ_CRASH("Unexpected type for LatentOp::Eqz"); + } + } + break; + } + } + + if (b->hasBlockResults()) { + freeResultRegisters(b->resultType); + } +} + +bool BaseCompiler::emitBranchPerform(BranchState* b) { + switch (latentType_.kind()) { + case ValType::I32: { + if (b->i32.rhsImm) { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i32.lhs, + Imm32(b->i32.imm))) { + return false; + } + } else { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i32.lhs, + b->i32.rhs)) { + return false; + } + freeI32(b->i32.rhs); + } + freeI32(b->i32.lhs); + break; + } + case ValType::I64: { + if (b->i64.rhsImm) { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i64.lhs, + Imm64(b->i64.imm))) { + return false; + } + } else { + if (!jumpConditionalWithResults(b, latentIntCmp_, b->i64.lhs, + b->i64.rhs)) { + return false; + } + freeI64(b->i64.rhs); + } + freeI64(b->i64.lhs); + break; + } + case ValType::F32: { + if (!jumpConditionalWithResults(b, latentDoubleCmp_, b->f32.lhs, + b->f32.rhs)) { + return false; + } + freeF32(b->f32.lhs); + freeF32(b->f32.rhs); + break; + } + case ValType::F64: { + if (!jumpConditionalWithResults(b, latentDoubleCmp_, b->f64.lhs, + b->f64.rhs)) { + return false; + } + freeF64(b->f64.lhs); + freeF64(b->f64.rhs); + break; + } + default: { + MOZ_CRASH("Unexpected type for LatentOp::Compare"); + } + } + resetLatentOp(); + return true; +} + +// For blocks and loops and ifs: +// +// - Sync the value stack before going into the block in order to simplify exit +// from the block: all exits from the block can assume that there are no +// live registers except the one carrying the exit value. +// - The block can accumulate a number of dead values on the stacks, so when +// branching out of the block or falling out at the end be sure to +// pop the appropriate stacks back to where they were on entry, while +// preserving the exit value. +// - A continue branch in a loop is much like an exit branch, but the branch +// value must not be preserved. +// - The exit value is always in a designated join register (type dependent). + +bool BaseCompiler::emitBlock() { + ResultType params; + if (!iter_.readBlock(¶ms)) { + return false; + } + + if (!deadCode_) { + sync(); // Simplifies branching out from block + } + + initControl(controlItem(), params); + + return true; +} + +bool BaseCompiler::endBlock(ResultType type) { + Control& block = controlItem(); + + if (deadCode_) { + // Block does not fall through; reset stack. + fr.resetStackHeight(block.stackHeight, type); + popValueStackTo(block.stackSize); + } else { + // If the block label is used, we have a control join, so we need to shuffle + // fallthrough values into place. Otherwise if it's not a control join, we + // can leave the value stack alone. + MOZ_ASSERT(stk_.length() == block.stackSize + type.length()); + if (block.label.used()) { + popBlockResults(type, block.stackHeight, ContinuationKind::Fallthrough); + } + block.bceSafeOnExit &= bceSafe_; + } + + // Bind after cleanup: branches out will have popped the stack. + if (block.label.used()) { + masm.bind(&block.label); + if (deadCode_) { + captureResultRegisters(type); + deadCode_ = false; + } + if (!pushBlockResults(type)) { + return false; + } + } + + bceSafe_ = block.bceSafeOnExit; + + return true; +} + +bool BaseCompiler::emitLoop() { + ResultType params; + if (!iter_.readLoop(¶ms)) { + return false; + } + + if (!deadCode_) { + sync(); // Simplifies branching out from block + } + + initControl(controlItem(), params); + bceSafe_ = 0; + + if (!deadCode_) { + // Loop entry is a control join, so shuffle the entry parameters into the + // well-known locations. + if (!topBlockParams(params)) { + return false; + } + masm.nopAlign(CodeAlignment); + masm.bind(&controlItem(0).label); + // The interrupt check barfs if there are live registers. + sync(); + if (!addInterruptCheck()) { + return false; + } + } + + return true; +} + +// The bodies of the "then" and "else" arms can be arbitrary sequences +// of expressions, they push control and increment the nesting and can +// even be targeted by jumps. A branch to the "if" block branches to +// the exit of the if, ie, it's like "break". Consider: +// +// (func (result i32) +// (if (i32.const 1) +// (begin (br 1) (unreachable)) +// (begin (unreachable))) +// (i32.const 1)) +// +// The branch causes neither of the unreachable expressions to be +// evaluated. + +bool BaseCompiler::emitIf() { + ResultType params; + Nothing unused_cond; + if (!iter_.readIf(¶ms, &unused_cond)) { + return false; + } + + BranchState b(&controlItem().otherLabel, InvertBranch(true)); + if (!deadCode_) { + needResultRegisters(params); + emitBranchSetup(&b); + freeResultRegisters(params); + sync(); + } else { + resetLatentOp(); + } + + initControl(controlItem(), params); + + if (!deadCode_) { + // Because params can flow immediately to results in the case of an empty + // "then" or "else" block, and the result of an if/then is a join in + // general, we shuffle params eagerly to the result allocations. + if (!topBlockParams(params)) { + return false; + } + if (!emitBranchPerform(&b)) { + return false; + } + } + + return true; +} + +bool BaseCompiler::endIfThen(ResultType type) { + Control& ifThen = controlItem(); + + // The parameters to the "if" logically flow to both the "then" and "else" + // blocks, but the "else" block is empty. Since we know that the "if" + // type-checks, that means that the "else" parameters are the "else" results, + // and that the "if"'s result type is the same as its parameter type. + + if (deadCode_) { + // "then" arm does not fall through; reset stack. + fr.resetStackHeight(ifThen.stackHeight, type); + popValueStackTo(ifThen.stackSize); + if (!ifThen.deadOnArrival) { + captureResultRegisters(type); + } + } else { + MOZ_ASSERT(stk_.length() == ifThen.stackSize + type.length()); + // Assume we have a control join, so place results in block result + // allocations. + popBlockResults(type, ifThen.stackHeight, ContinuationKind::Fallthrough); + MOZ_ASSERT(!ifThen.deadOnArrival); + } + + if (ifThen.otherLabel.used()) { + masm.bind(&ifThen.otherLabel); + } + + if (ifThen.label.used()) { + masm.bind(&ifThen.label); + } + + if (!deadCode_) { + ifThen.bceSafeOnExit &= bceSafe_; + } + + deadCode_ = ifThen.deadOnArrival; + if (!deadCode_) { + if (!pushBlockResults(type)) { + return false; + } + } + + bceSafe_ = ifThen.bceSafeOnExit & ifThen.bceSafeOnEntry; + + return true; +} + +bool BaseCompiler::emitElse() { + ResultType params, results; + BaseNothingVector unused_thenValues{}; + + if (!iter_.readElse(¶ms, &results, &unused_thenValues)) { + return false; + } + + Control& ifThenElse = controlItem(0); + + // See comment in endIfThenElse, below. + + // Exit the "then" branch. + + ifThenElse.deadThenBranch = deadCode_; + + if (deadCode_) { + fr.resetStackHeight(ifThenElse.stackHeight, results); + popValueStackTo(ifThenElse.stackSize); + } else { + MOZ_ASSERT(stk_.length() == ifThenElse.stackSize + results.length()); + popBlockResults(results, ifThenElse.stackHeight, ContinuationKind::Jump); + freeResultRegisters(results); + MOZ_ASSERT(!ifThenElse.deadOnArrival); + } + + if (!deadCode_) { + masm.jump(&ifThenElse.label); + } + + if (ifThenElse.otherLabel.used()) { + masm.bind(&ifThenElse.otherLabel); + } + + // Reset to the "else" branch. + + if (!deadCode_) { + ifThenElse.bceSafeOnExit &= bceSafe_; + } + + deadCode_ = ifThenElse.deadOnArrival; + bceSafe_ = ifThenElse.bceSafeOnEntry; + + fr.resetStackHeight(ifThenElse.stackHeight, params); + + if (!deadCode_) { + captureResultRegisters(params); + if (!pushBlockResults(params)) { + return false; + } + } + + return true; +} + +bool BaseCompiler::endIfThenElse(ResultType type) { + Control& ifThenElse = controlItem(); + + // The expression type is not a reliable guide to what we'll find + // on the stack, we could have (if E (i32.const 1) (unreachable)) + // in which case the "else" arm is AnyType but the type of the + // full expression is I32. So restore whatever's there, not what + // we want to find there. The "then" arm has the same constraint. + + if (deadCode_) { + // "then" arm does not fall through; reset stack. + fr.resetStackHeight(ifThenElse.stackHeight, type); + popValueStackTo(ifThenElse.stackSize); + } else { + MOZ_ASSERT(stk_.length() == ifThenElse.stackSize + type.length()); + // Assume we have a control join, so place results in block result + // allocations. + popBlockResults(type, ifThenElse.stackHeight, + ContinuationKind::Fallthrough); + ifThenElse.bceSafeOnExit &= bceSafe_; + MOZ_ASSERT(!ifThenElse.deadOnArrival); + } + + if (ifThenElse.label.used()) { + masm.bind(&ifThenElse.label); + } + + bool joinLive = + !ifThenElse.deadOnArrival && + (!ifThenElse.deadThenBranch || !deadCode_ || ifThenElse.label.bound()); + + if (joinLive) { + // No values were provided by the "then" path, but capture the values + // provided by the "else" path. + if (deadCode_) { + captureResultRegisters(type); + } + deadCode_ = false; + } + + bceSafe_ = ifThenElse.bceSafeOnExit; + + if (!deadCode_) { + if (!pushBlockResults(type)) { + return false; + } + } + + return true; +} + +bool BaseCompiler::emitEnd() { + LabelKind kind; + ResultType type; + BaseNothingVector unused_values{}; + if (!iter_.readEnd(&kind, &type, &unused_values, &unused_values)) { + return false; + } + + // Every label case is responsible to pop the control item at the appropriate + // time for the label case + switch (kind) { + case LabelKind::Body: + if (!endBlock(type)) { + return false; + } + doReturn(ContinuationKind::Fallthrough); + // This is emitted here after `doReturn` to avoid being executed in the + // normal return path of a function, and instead only when a `delegate` + // jumps to it. + if (!emitBodyDelegateThrowPad()) { + return false; + } + iter_.popEnd(); + MOZ_ASSERT(iter_.controlStackEmpty()); + return iter_.endFunction(iter_.end()); + case LabelKind::Block: + if (!endBlock(type)) { + return false; + } + iter_.popEnd(); + break; + case LabelKind::Loop: + // The end of a loop isn't a branch target, so we can just leave its + // results on the expression stack to be consumed by the outer block. + iter_.popEnd(); + break; + case LabelKind::Then: + if (!endIfThen(type)) { + return false; + } + iter_.popEnd(); + break; + case LabelKind::Else: + if (!endIfThenElse(type)) { + return false; + } + iter_.popEnd(); + break; + case LabelKind::Try: + case LabelKind::Catch: + case LabelKind::CatchAll: + if (!endTryCatch(type)) { + return false; + } + iter_.popEnd(); + break; + } + + return true; +} + +bool BaseCompiler::emitBr() { + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + if (!iter_.readBr(&relativeDepth, &type, &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + // Save any values in the designated join registers, as if the target block + // returned normally. + + popBlockResults(type, target.stackHeight, ContinuationKind::Jump); + masm.jump(&target.label); + + // The registers holding the join values are free for the remainder of this + // block. + + freeResultRegisters(type); + + deadCode_ = true; + + return true; +} + +bool BaseCompiler::emitBrIf() { + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + Nothing unused_condition; + if (!iter_.readBrIf(&relativeDepth, &type, &unused_values, + &unused_condition)) { + return false; + } + + if (deadCode_) { + resetLatentOp(); + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), type); + emitBranchSetup(&b); + return emitBranchPerform(&b); +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +bool BaseCompiler::emitBrOnNull() { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + Nothing unused_condition; + if (!iter_.readBrOnNull(&relativeDepth, &type, &unused_values, + &unused_condition)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), type); + if (b.hasBlockResults()) { + needResultRegisters(b.resultType); + } + RegRef rp = popRef(); + if (b.hasBlockResults()) { + freeResultRegisters(b.resultType); + } + if (!jumpConditionalWithResults(&b, Assembler::Equal, rp, + ImmWord(NULLREF_VALUE))) { + return false; + } + pushRef(rp); + + return true; +} + +bool BaseCompiler::emitBrOnNonNull() { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t relativeDepth; + ResultType type; + BaseNothingVector unused_values{}; + Nothing unused_condition; + if (!iter_.readBrOnNonNull(&relativeDepth, &type, &unused_values, + &unused_condition)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(relativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), type); + MOZ_ASSERT(b.hasBlockResults(), "br_on_non_null has block results"); + + // Don't allocate the result register used in the branch + needIntegerResultRegisters(b.resultType); + + // Get the ref from the top of the stack + RegRef condition = popRef(); + + // Create a copy of the ref for passing to the on_non_null label, + // the original ref is used in the condition. + RegRef rp = needRef(); + moveRef(condition, rp); + pushRef(rp); + + freeIntegerResultRegisters(b.resultType); + + if (!jumpConditionalWithResults(&b, Assembler::NotEqual, condition, + ImmWord(NULLREF_VALUE))) { + return false; + } + + freeRef(condition); + + // Dropping null reference. + dropValue(); + + return true; +} +#endif + +bool BaseCompiler::emitBrTable() { + Uint32Vector depths; + uint32_t defaultDepth; + ResultType branchParams; + BaseNothingVector unused_values{}; + Nothing unused_index; + // N.B., `branchParams' gets set to the type of the default branch target. In + // the presence of subtyping, it could be that the different branch targets + // have different types. Here we rely on the assumption that the value + // representations (e.g. Stk value types) of all branch target types are the + // same, in the baseline compiler. Notably, this means that all Ref types + // should be represented the same. + if (!iter_.readBrTable(&depths, &defaultDepth, &branchParams, &unused_values, + &unused_index)) { + return false; + } + + if (deadCode_) { + return true; + } + + // Don't use param registers for rc + needIntegerResultRegisters(branchParams); + + // Table switch value always on top. + RegI32 rc = popI32(); + + freeIntegerResultRegisters(branchParams); + + StackHeight resultsBase(0); + if (!topBranchParams(branchParams, &resultsBase)) { + return false; + } + + Label dispatchCode; + masm.branch32(Assembler::Below, rc, Imm32(depths.length()), &dispatchCode); + + // This is the out-of-range stub. rc is dead here but we don't need it. + + shuffleStackResultsBeforeBranch( + resultsBase, controlItem(defaultDepth).stackHeight, branchParams); + controlItem(defaultDepth).bceSafeOnExit &= bceSafe_; + masm.jump(&controlItem(defaultDepth).label); + + // Emit stubs. rc is dead in all of these but we don't need it. + // + // The labels in the vector are in the TempAllocator and will + // be freed by and by. + // + // TODO / OPTIMIZE (Bug 1316804): Branch directly to the case code if we + // can, don't emit an intermediate stub. + + LabelVector stubs; + if (!stubs.reserve(depths.length())) { + return false; + } + + for (uint32_t depth : depths) { + stubs.infallibleEmplaceBack(NonAssertingLabel()); + masm.bind(&stubs.back()); + shuffleStackResultsBeforeBranch(resultsBase, controlItem(depth).stackHeight, + branchParams); + controlItem(depth).bceSafeOnExit &= bceSafe_; + masm.jump(&controlItem(depth).label); + } + + // Emit table. + + Label theTable; + jumpTable(stubs, &theTable); + + // Emit indirect jump. rc is live here. + + tableSwitch(&theTable, rc, &dispatchCode); + + deadCode_ = true; + + // Clean up. + + freeI32(rc); + popValueStackBy(branchParams.length()); + + return true; +} + +bool BaseCompiler::emitTry() { + ResultType params; + if (!iter_.readTry(¶ms)) { + return false; + } + + if (!deadCode_) { + // Simplifies jumping out, but it is also necessary so that control + // can re-enter the catch handler without restoring registers. + sync(); + } + + initControl(controlItem(), params); + + if (!deadCode_) { + // Be conservative for BCE due to complex control flow in try blocks. + controlItem().bceSafeOnExit = 0; + if (!startTryNote(&controlItem().tryNoteIndex)) { + return false; + } + } + + return true; +} + +void BaseCompiler::emitCatchSetup(LabelKind kind, Control& tryCatch, + const ResultType& resultType) { + // Catch ends the try or last catch, so we finish this like endIfThen. + if (deadCode_) { + fr.resetStackHeight(tryCatch.stackHeight, resultType); + popValueStackTo(tryCatch.stackSize); + } else { + // If the previous block is a catch, we need to handle the extra exception + // reference on the stack (for rethrow) and thus the stack size is 1 more. + MOZ_ASSERT(stk_.length() == tryCatch.stackSize + resultType.length() + + (kind == LabelKind::Try ? 0 : 1)); + // Try jumps to the end of the try-catch block unless a throw is done. + if (kind == LabelKind::Try) { + popBlockResults(resultType, tryCatch.stackHeight, ContinuationKind::Jump); + } else { + popCatchResults(resultType, tryCatch.stackHeight); + } + MOZ_ASSERT(stk_.length() == tryCatch.stackSize); + freeResultRegisters(resultType); + MOZ_ASSERT(!tryCatch.deadOnArrival); + } + + // Reset to this "catch" branch. + deadCode_ = tryCatch.deadOnArrival; + + // We use the empty result type here because catch does *not* take the + // try-catch block parameters. + fr.resetStackHeight(tryCatch.stackHeight, ResultType::Empty()); + + if (deadCode_) { + return; + } + + bceSafe_ = 0; + + // The end of the previous try/catch jumps to the join point. + masm.jump(&tryCatch.label); + + // Note end of try block for finding the catch block target. This needs + // to happen after the stack is reset to the correct height. + if (kind == LabelKind::Try) { + finishTryNote(controlItem().tryNoteIndex); + } +} + +bool BaseCompiler::emitCatch() { + LabelKind kind; + uint32_t tagIndex; + ResultType paramType, resultType; + BaseNothingVector unused_tryValues{}; + + if (!iter_.readCatch(&kind, &tagIndex, ¶mType, &resultType, + &unused_tryValues)) { + return false; + } + + Control& tryCatch = controlItem(); + + emitCatchSetup(kind, tryCatch, resultType); + + if (deadCode_) { + return true; + } + + // Construct info used for the exception landing pad. + CatchInfo catchInfo(tagIndex); + if (!tryCatch.catchInfos.emplaceBack(catchInfo)) { + return false; + } + + masm.bind(&tryCatch.catchInfos.back().label); + + // Extract the arguments in the exception package and push them. + const SharedTagType& tagType = moduleEnv_.tags[tagIndex].type; + const ValTypeVector& params = tagType->argTypes_; + const TagOffsetVector& offsets = tagType->argOffsets_; + + // The landing pad uses the block return protocol to communicate the + // exception object pointer to the catch block. + ResultType exnResult = ResultType::Single(RefType::extern_()); + captureResultRegisters(exnResult); + if (!pushBlockResults(exnResult)) { + return false; + } + RegRef exn = popRef(); + RegPtr data = needPtr(); + + masm.loadPtr(Address(exn, (int32_t)WasmExceptionObject::offsetOfData()), + data); + + // This method can increase stk_.length() by an unbounded amount, so we need + // to perform an allocation here to accomodate the variable number of values. + // There is enough headroom for the fixed number of values. The general case + // is handled in emitBody. + if (!stk_.reserve(stk_.length() + params.length() + 1)) { + return false; + } + + // This reference is pushed onto the stack because a potential rethrow + // may need to access it. It is always popped at the end of the block. + pushRef(exn); + + for (uint32_t i = 0; i < params.length(); i++) { + int32_t offset = offsets[i]; + switch (params[i].kind()) { + case ValType::I32: { + RegI32 reg = needI32(); + masm.load32(Address(data, offset), reg); + pushI32(reg); + break; + } + case ValType::I64: { + RegI64 reg = needI64(); + masm.load64(Address(data, offset), reg); + pushI64(reg); + break; + } + case ValType::F32: { + RegF32 reg = needF32(); + masm.loadFloat32(Address(data, offset), reg); + pushF32(reg); + break; + } + case ValType::F64: { + RegF64 reg = needF64(); + masm.loadDouble(Address(data, offset), reg); + pushF64(reg); + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + RegV128 reg = needV128(); + masm.loadUnalignedSimd128(Address(data, offset), reg); + pushV128(reg); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::Ref: { + // TODO/AnyRef-boxing: With boxed immediates and strings, this may need + // to handle other kinds of values. + ASSERT_ANYREF_IS_JSOBJECT; + RegRef reg = needRef(); + masm.loadPtr(Address(data, offset), reg); + pushRef(reg); + break; + } + } + } + freePtr(data); + + return true; +} + +bool BaseCompiler::emitCatchAll() { + LabelKind kind; + ResultType paramType, resultType; + BaseNothingVector unused_tryValues{}; + + if (!iter_.readCatchAll(&kind, ¶mType, &resultType, &unused_tryValues)) { + return false; + } + + Control& tryCatch = controlItem(); + + emitCatchSetup(kind, tryCatch, resultType); + + if (deadCode_) { + return true; + } + + CatchInfo catchInfo(CatchAllIndex); + if (!tryCatch.catchInfos.emplaceBack(catchInfo)) { + return false; + } + + masm.bind(&tryCatch.catchInfos.back().label); + + // The landing pad uses the block return protocol to communicate the + // exception object pointer to the catch block. + ResultType exnResult = ResultType::Single(RefType::extern_()); + captureResultRegisters(exnResult); + // This reference is pushed onto the stack because a potential rethrow + // may need to access it. It is always popped at the end of the block. + return pushBlockResults(exnResult); +} + +bool BaseCompiler::emitBodyDelegateThrowPad() { + Control& block = controlItem(); + + // Only emit a landing pad if a `delegate` has generated a jump to here. + if (block.otherLabel.used()) { + StackHeight savedHeight = fr.stackHeight(); + fr.setStackHeight(block.stackHeight); + masm.bind(&block.otherLabel); + + // A try-delegate jumps immediately to its delegated try block, so we are + // responsible to unpack the exception and rethrow it. + RegRef exn; + RegRef tag; + consumePendingException(&exn, &tag); + freeRef(tag); + if (!throwFrom(exn)) { + return false; + } + fr.setStackHeight(savedHeight); + } + + return true; +} + +bool BaseCompiler::emitDelegate() { + uint32_t relativeDepth; + ResultType resultType; + BaseNothingVector unused_tryValues{}; + + if (!iter_.readDelegate(&relativeDepth, &resultType, &unused_tryValues)) { + return false; + } + + Control& tryDelegate = controlItem(); + + // End the try branch like a plain catch block without exception ref handling. + if (deadCode_) { + fr.resetStackHeight(tryDelegate.stackHeight, resultType); + popValueStackTo(tryDelegate.stackSize); + } else { + MOZ_ASSERT(stk_.length() == tryDelegate.stackSize + resultType.length()); + popBlockResults(resultType, tryDelegate.stackHeight, + ContinuationKind::Jump); + freeResultRegisters(resultType); + masm.jump(&tryDelegate.label); + MOZ_ASSERT(!tryDelegate.deadOnArrival); + } + + deadCode_ = tryDelegate.deadOnArrival; + + if (deadCode_) { + return true; + } + + // Create an exception landing pad that immediately branches to the landing + // pad of the delegated try block. + masm.bind(&tryDelegate.otherLabel); + + StackHeight savedHeight = fr.stackHeight(); + fr.setStackHeight(tryDelegate.stackHeight); + + // Mark the end of the try body. This may insert a nop. + finishTryNote(controlItem().tryNoteIndex); + + // The landing pad begins at this point + TryNoteVector& tryNotes = masm.tryNotes(); + TryNote& tryNote = tryNotes[controlItem().tryNoteIndex]; + tryNote.setLandingPad(masm.currentOffset(), masm.framePushed()); + + // Store the Instance that was left in InstanceReg by the exception + // handling mechanism, that is this frame's Instance but with the exception + // filled in Instance::pendingException. + fr.storeInstancePtr(InstanceReg); + + // If the target block is a non-try block, skip over it and find the next + // try block or the very last block (to re-throw out of the function). + Control& lastBlock = controlOutermost(); + while (controlKind(relativeDepth) != LabelKind::Try && + &controlItem(relativeDepth) != &lastBlock) { + relativeDepth++; + } + Control& target = controlItem(relativeDepth); + + popBlockResults(ResultType::Empty(), target.stackHeight, + ContinuationKind::Jump); + masm.jump(&target.otherLabel); + + fr.setStackHeight(savedHeight); + + // Where the try branch jumps to, if it's not dead. + if (tryDelegate.label.used()) { + masm.bind(&tryDelegate.label); + } + + captureResultRegisters(resultType); + bceSafe_ = tryDelegate.bceSafeOnExit; + + return pushBlockResults(resultType); +} + +bool BaseCompiler::endTryCatch(ResultType type) { + Control& tryCatch = controlItem(); + LabelKind tryKind = controlKind(0); + + if (deadCode_) { + fr.resetStackHeight(tryCatch.stackHeight, type); + popValueStackTo(tryCatch.stackSize); + } else { + // If the previous block is a catch, we must handle the extra exception + // reference on the stack (for rethrow) and thus the stack size is 1 more. + MOZ_ASSERT(stk_.length() == tryCatch.stackSize + type.length() + + (tryKind == LabelKind::Try ? 0 : 1)); + // Assume we have a control join, so place results in block result + // allocations and also handle the implicit exception reference if needed. + if (tryKind == LabelKind::Try) { + popBlockResults(type, tryCatch.stackHeight, ContinuationKind::Jump); + } else { + popCatchResults(type, tryCatch.stackHeight); + } + MOZ_ASSERT(stk_.length() == tryCatch.stackSize); + // Since we will emit a landing pad after this and jump over it to get to + // the control join, we free these here and re-capture at the join. + freeResultRegisters(type); + masm.jump(&tryCatch.label); + MOZ_ASSERT(!tryCatch.bceSafeOnExit); + MOZ_ASSERT(!tryCatch.deadOnArrival); + } + + deadCode_ = tryCatch.deadOnArrival; + + if (deadCode_) { + return true; + } + + // Create landing pad for all catch handlers in this block. + // When used for a catchless try block, this will generate a landing pad + // with no handlers and only the fall-back rethrow. + masm.bind(&tryCatch.otherLabel); + + // The stack height also needs to be set not for a block result, but for the + // entry to the exception handlers. This is reset again below for the join. + StackHeight prePadHeight = fr.stackHeight(); + fr.setStackHeight(tryCatch.stackHeight); + + // If we are in a catchless try block, then there were no catch blocks to + // mark the end of the try note, so we need to end it here. + if (tryKind == LabelKind::Try) { + // Mark the end of the try body. This may insert a nop. + finishTryNote(controlItem().tryNoteIndex); + } + + // The landing pad begins at this point + TryNoteVector& tryNotes = masm.tryNotes(); + TryNote& tryNote = tryNotes[controlItem().tryNoteIndex]; + tryNote.setLandingPad(masm.currentOffset(), masm.framePushed()); + + // Store the Instance that was left in InstanceReg by the exception + // handling mechanism, that is this frame's Instance but with the exception + // filled in Instance::pendingException. + fr.storeInstancePtr(InstanceReg); + + // Load exception pointer from Instance and make sure that it is + // saved before the following call will clear it. + RegRef exn; + RegRef tag; + consumePendingException(&exn, &tag); + + // Get a register to hold the tags for each catch + RegRef catchTag = needRef(); + + // Ensure that the exception is assigned to the block return register + // before branching to a handler. + pushRef(exn); + ResultType exnResult = ResultType::Single(RefType::extern_()); + popBlockResults(exnResult, tryCatch.stackHeight, ContinuationKind::Jump); + freeResultRegisters(exnResult); + + bool hasCatchAll = false; + for (CatchInfo& info : tryCatch.catchInfos) { + if (info.tagIndex != CatchAllIndex) { + MOZ_ASSERT(!hasCatchAll); + loadTag(RegPtr(InstanceReg), info.tagIndex, catchTag); + masm.branchPtr(Assembler::Equal, tag, catchTag, &info.label); + } else { + masm.jump(&info.label); + hasCatchAll = true; + } + } + freeRef(catchTag); + freeRef(tag); + + // If none of the tag checks succeed and there is no catch_all, + // then we rethrow the exception. + if (!hasCatchAll) { + captureResultRegisters(exnResult); + if (!pushBlockResults(exnResult) || !throwFrom(popRef())) { + return false; + } + } + + // Reset stack height for join. + fr.setStackHeight(prePadHeight); + + // Create join point. + if (tryCatch.label.used()) { + masm.bind(&tryCatch.label); + } + + captureResultRegisters(type); + deadCode_ = tryCatch.deadOnArrival; + bceSafe_ = tryCatch.bceSafeOnExit; + + return pushBlockResults(type); +} + +bool BaseCompiler::emitThrow() { + uint32_t tagIndex; + BaseNothingVector unused_argValues{}; + + if (!iter_.readThrow(&tagIndex, &unused_argValues)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TagDesc& tagDesc = moduleEnv_.tags[tagIndex]; + const ResultType& params = tagDesc.type->resultType(); + const TagOffsetVector& offsets = tagDesc.type->argOffsets_; + + // Load the tag object +#ifdef RABALDR_PIN_INSTANCE + RegPtr instance(InstanceReg); +#else + RegPtr instance = needPtr(); + fr.loadInstancePtr(instance); +#endif + RegRef tag = needRef(); + loadTag(instance, tagIndex, tag); +#ifndef RABALDR_PIN_INSTANCE + freePtr(instance); +#endif + + // Create the new exception object that we will throw. + pushRef(tag); + if (!emitInstanceCall(SASigExceptionNew)) { + return false; + } + + // Get registers for exn and data, excluding the prebarrier register + needPtr(RegPtr(PreBarrierReg)); + RegRef exn = popRef(); + RegPtr data = needPtr(); + freePtr(RegPtr(PreBarrierReg)); + + masm.loadPtr(Address(exn, WasmExceptionObject::offsetOfData()), data); + + for (int32_t i = params.length() - 1; i >= 0; i--) { + uint32_t offset = offsets[i]; + switch (params[i].kind()) { + case ValType::I32: { + RegI32 reg = popI32(); + masm.store32(reg, Address(data, offset)); + freeI32(reg); + break; + } + case ValType::I64: { + RegI64 reg = popI64(); + masm.store64(reg, Address(data, offset)); + freeI64(reg); + break; + } + case ValType::F32: { + RegF32 reg = popF32(); + masm.storeFloat32(reg, Address(data, offset)); + freeF32(reg); + break; + } + case ValType::F64: { + RegF64 reg = popF64(); + masm.storeDouble(reg, Address(data, offset)); + freeF64(reg); + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + RegV128 reg = popV128(); + masm.storeUnalignedSimd128(reg, Address(data, offset)); + freeV128(reg); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::Ref: { + RegPtr valueAddr(PreBarrierReg); + needPtr(valueAddr); + masm.computeEffectiveAddress(Address(data, offset), valueAddr); + RegRef rv = popRef(); + pushPtr(data); + // emitBarrieredStore preserves exn, rv + if (!emitBarrieredStore(Some(exn), valueAddr, rv, + PreBarrierKind::Normal, + PostBarrierKind::Imprecise)) { + return false; + } + popPtr(data); + freeRef(rv); + break; + } + } + } + freePtr(data); + + deadCode_ = true; + + return throwFrom(exn); +} + +bool BaseCompiler::emitRethrow() { + uint32_t relativeDepth; + if (!iter_.readRethrow(&relativeDepth)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& tryCatch = controlItem(relativeDepth); + RegRef exn = needRef(); + peekRefAt(tryCatch.stackSize, exn); + + deadCode_ = true; + + return throwFrom(exn); +} + +bool BaseCompiler::emitDrop() { + if (!iter_.readDrop()) { + return false; + } + + if (deadCode_) { + return true; + } + + dropValue(); + return true; +} + +void BaseCompiler::doReturn(ContinuationKind kind) { + if (deadCode_) { + return; + } + + StackHeight height = controlOutermost().stackHeight; + ResultType type = ResultType::Vector(funcType().results()); + popBlockResults(type, height, kind); + masm.jump(&returnLabel_); + freeResultRegisters(type); +} + +bool BaseCompiler::emitReturn() { + BaseNothingVector unused_values{}; + if (!iter_.readReturn(&unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + doReturn(ContinuationKind::Jump); + deadCode_ = true; + + return true; +} + +bool BaseCompiler::emitCallArgs(const ValTypeVector& argTypes, + const StackResultsLoc& results, + FunctionCall* baselineCall, + CalleeOnStack calleeOnStack) { + MOZ_ASSERT(!deadCode_); + + ArgTypeVector args(argTypes, results.stackResults()); + uint32_t naturalArgCount = argTypes.length(); + uint32_t abiArgCount = args.lengthWithStackResults(); + startCallArgs(StackArgAreaSizeUnaligned(args), baselineCall); + + // Args are deeper on the stack than the stack result area, if any. + size_t argsDepth = results.count(); + // They're deeper than the callee too, for callIndirect. + if (calleeOnStack == CalleeOnStack::True) { + argsDepth++; + } + + for (size_t i = 0; i < abiArgCount; ++i) { + if (args.isNaturalArg(i)) { + size_t naturalIndex = args.naturalIndex(i); + size_t stackIndex = naturalArgCount - 1 - naturalIndex + argsDepth; + passArg(argTypes[naturalIndex], peek(stackIndex), baselineCall); + } else { + // The synthetic stack result area pointer. + ABIArg argLoc = baselineCall->abi.next(MIRType::Pointer); + if (argLoc.kind() == ABIArg::Stack) { + ScratchPtr scratch(*this); + fr.computeOutgoingStackResultAreaPtr(results, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + argLoc.offsetFromArgBase())); + } else { + fr.computeOutgoingStackResultAreaPtr(results, RegPtr(argLoc.gpr())); + } + } + } + +#ifndef RABALDR_PIN_INSTANCE + fr.loadInstancePtr(InstanceReg); +#endif + return true; +} + +void BaseCompiler::pushReturnValueOfCall(const FunctionCall& call, + MIRType type) { + switch (type) { + case MIRType::Int32: { + RegI32 rv = captureReturnedI32(); + pushI32(rv); + break; + } + case MIRType::Int64: { + RegI64 rv = captureReturnedI64(); + pushI64(rv); + break; + } + case MIRType::Float32: { + RegF32 rv = captureReturnedF32(call); + pushF32(rv); + break; + } + case MIRType::Double: { + RegF64 rv = captureReturnedF64(call); + pushF64(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: { + RegV128 rv = captureReturnedV128(call); + pushV128(rv); + break; + } +#endif + case MIRType::RefOrNull: { + RegRef rv = captureReturnedRef(); + pushRef(rv); + break; + } + default: + // In particular, passing |type| as MIRType::Void or MIRType::Pointer to + // this function is an error. + MOZ_CRASH("Function return type"); + } +} + +bool BaseCompiler::pushStackResultsForCall(const ResultType& type, RegPtr temp, + StackResultsLoc* loc) { + if (!ABIResultIter::HasStackResults(type)) { + return true; + } + + // This method can increase stk_.length() by an unbounded amount, so we need + // to perform an allocation here to accomodate the variable number of values. + // There is enough headroom for any fixed number of values. The general case + // is handled in emitBody. + if (!stk_.reserve(stk_.length() + type.length())) { + return false; + } + + // Measure stack results. + ABIResultIter i(type); + size_t count = 0; + for (; !i.done(); i.next()) { + if (i.cur().onStack()) { + count++; + } + } + uint32_t bytes = i.stackBytesConsumedSoFar(); + + // Reserve space for the stack results. + StackHeight resultsBase = fr.stackHeight(); + uint32_t height = fr.prepareStackResultArea(resultsBase, bytes); + + // Push Stk values onto the value stack, and zero out Ref values. + for (i.switchToPrev(); !i.done(); i.prev()) { + const ABIResult& result = i.cur(); + if (result.onStack()) { + Stk v = captureStackResult(result, resultsBase, bytes); + push(v); + if (v.kind() == Stk::MemRef) { + stackMapGenerator_.memRefsOnStk++; + fr.storeImmediatePtrToStack(intptr_t(0), v.offs(), temp); + } + } + } + + *loc = StackResultsLoc(bytes, count, height); + + return true; +} + +// After a call, some results may be written to the stack result locations that +// are pushed on the machine stack after any stack args. If there are stack +// args and stack results, these results need to be shuffled down, as the args +// are "consumed" by the call. +void BaseCompiler::popStackResultsAfterCall(const StackResultsLoc& results, + uint32_t stackArgBytes) { + if (results.bytes() != 0) { + popValueStackBy(results.count()); + if (stackArgBytes != 0) { + uint32_t srcHeight = results.height(); + MOZ_ASSERT(srcHeight >= stackArgBytes + results.bytes()); + uint32_t destHeight = srcHeight - stackArgBytes; + + fr.shuffleStackResultsTowardFP(srcHeight, destHeight, results.bytes(), + ABINonArgReturnVolatileReg); + } + } +} + +// For now, always sync() at the beginning of the call to easily save live +// values. +// +// TODO / OPTIMIZE (Bug 1316806): We may be able to avoid a full sync(), since +// all we want is to save live registers that won't be saved by the callee or +// that we need for outgoing args - we don't need to sync the locals. We can +// just push the necessary registers, it'll be like a lightweight sync. +// +// Even some of the pushing may be unnecessary if the registers will be consumed +// by the call, because then what we want is parallel assignment to the argument +// registers or onto the stack for outgoing arguments. A sync() is just +// simpler. + +bool BaseCompiler::emitCall() { + uint32_t funcIndex; + BaseNothingVector args_{}; + if (!iter_.readCall(&funcIndex, &args_)) { + return false; + } + + if (deadCode_) { + return true; + } + + sync(); + + const FuncType& funcType = *moduleEnv_.funcs[funcIndex].type; + bool import = moduleEnv_.funcIsImport(funcIndex); + + uint32_t numArgs = funcType.args().length(); + size_t stackArgBytes = stackConsumed(numArgs); + + ResultType resultType(ResultType::Vector(funcType.results())); + StackResultsLoc results; + if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) { + return false; + } + + FunctionCall baselineCall{}; + beginCall(baselineCall, UseABI::Wasm, + import ? RestoreRegisterStateAndRealm::True + : RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(funcType.args(), results, &baselineCall, + CalleeOnStack::False)) { + return false; + } + + CodeOffset raOffset; + if (import) { + raOffset = callImport(moduleEnv_.offsetOfFuncImportInstanceData(funcIndex), + baselineCall); + } else { + raOffset = callDefinition(funcIndex, baselineCall); + } + + if (!createStackMap("emitCall", raOffset)) { + return false; + } + + popStackResultsAfterCall(results, stackArgBytes); + + endCall(baselineCall, stackArgBytes); + + popValueStackBy(numArgs); + + captureCallResultRegisters(resultType); + return pushCallResults(baselineCall, resultType, results); +} + +bool BaseCompiler::emitCallIndirect() { + uint32_t funcTypeIndex; + uint32_t tableIndex; + Nothing callee_; + BaseNothingVector args_{}; + if (!iter_.readCallIndirect(&funcTypeIndex, &tableIndex, &callee_, &args_)) { + return false; + } + + if (deadCode_) { + return true; + } + + sync(); + + const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType(); + + // Stack: ... arg1 .. argn callee + + uint32_t numArgs = funcType.args().length() + 1; + size_t stackArgBytes = stackConsumed(numArgs); + + ResultType resultType(ResultType::Vector(funcType.results())); + StackResultsLoc results; + if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) { + return false; + } + + FunctionCall baselineCall{}; + // State and realm are restored as needed by by callIndirect (really by + // MacroAssembler::wasmCallIndirect). + beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(funcType.args(), results, &baselineCall, + CalleeOnStack::True)) { + return false; + } + + const Stk& callee = peek(results.count()); + CodeOffset fastCallOffset; + CodeOffset slowCallOffset; + if (!callIndirect(funcTypeIndex, tableIndex, callee, baselineCall, + &fastCallOffset, &slowCallOffset)) { + return false; + } + if (!createStackMap("emitCallIndirect", fastCallOffset)) { + return false; + } + if (!createStackMap("emitCallIndirect", slowCallOffset)) { + return false; + } + + popStackResultsAfterCall(results, stackArgBytes); + + endCall(baselineCall, stackArgBytes); + + popValueStackBy(numArgs); + + captureCallResultRegisters(resultType); + return pushCallResults(baselineCall, resultType, results); +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +bool BaseCompiler::emitCallRef() { + const FuncType* funcType; + Nothing unused_callee; + BaseNothingVector unused_args{}; + if (!iter_.readCallRef(&funcType, &unused_callee, &unused_args)) { + return false; + } + + if (deadCode_) { + return true; + } + + sync(); + + // Stack: ... arg1 .. argn callee + + uint32_t numArgs = funcType->args().length() + 1; + size_t stackArgBytes = stackConsumed(numArgs); + + ResultType resultType(ResultType::Vector(funcType->results())); + StackResultsLoc results; + if (!pushStackResultsForCall(resultType, RegPtr(ABINonArgReg0), &results)) { + return false; + } + + FunctionCall baselineCall{}; + // State and realm are restored as needed by by callRef (really by + // MacroAssembler::wasmCallRef). + beginCall(baselineCall, UseABI::Wasm, RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(funcType->args(), results, &baselineCall, + CalleeOnStack::True)) { + return false; + } + + const Stk& callee = peek(results.count()); + CodeOffset fastCallOffset; + CodeOffset slowCallOffset; + callRef(callee, baselineCall, &fastCallOffset, &slowCallOffset); + if (!createStackMap("emitCallRef", fastCallOffset)) { + return false; + } + if (!createStackMap("emitCallRef", slowCallOffset)) { + return false; + } + + popStackResultsAfterCall(results, stackArgBytes); + + endCall(baselineCall, stackArgBytes); + + popValueStackBy(numArgs); + + captureCallResultRegisters(resultType); + return pushCallResults(baselineCall, resultType, results); +} +#endif + +void BaseCompiler::emitRound(RoundingMode roundingMode, ValType operandType) { + if (operandType == ValType::F32) { + RegF32 f0 = popF32(); + roundF32(roundingMode, f0); + pushF32(f0); + } else if (operandType == ValType::F64) { + RegF64 f0 = popF64(); + roundF64(roundingMode, f0); + pushF64(f0); + } else { + MOZ_CRASH("unexpected type"); + } +} + +bool BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, + ValType operandType) { + Nothing operand_; + if (!iter_.readUnary(operandType, &operand_)) { + return false; + } + + if (deadCode_) { + return true; + } + + RoundingMode roundingMode; + if (IsRoundingFunction(callee, &roundingMode) && + supportsRoundInstruction(roundingMode)) { + emitRound(roundingMode, operandType); + return true; + } + + sync(); + + ValTypeVector& signature = operandType == ValType::F32 ? SigF_ : SigD_; + ValType retType = operandType; + uint32_t numArgs = signature.length(); + size_t stackSpace = stackConsumed(numArgs); + StackResultsLoc noStackResults; + + FunctionCall baselineCall{}; + beginCall(baselineCall, UseABI::Builtin, RestoreRegisterStateAndRealm::False); + + if (!emitCallArgs(signature, noStackResults, &baselineCall, + CalleeOnStack::False)) { + return false; + } + + CodeOffset raOffset = builtinCall(callee, baselineCall); + if (!createStackMap("emitUnaryMathBuiltin[..]", raOffset)) { + return false; + } + + endCall(baselineCall, stackSpace); + + popValueStackBy(numArgs); + + pushReturnValueOfCall(baselineCall, retType.toMIRType()); + + return true; +} + +#ifdef RABALDR_INT_DIV_I64_CALLOUT +bool BaseCompiler::emitDivOrModI64BuiltinCall(SymbolicAddress callee, + ValType operandType) { + MOZ_ASSERT(operandType == ValType::I64); + MOZ_ASSERT(!deadCode_); + + sync(); + + needI64(specific_.abiReturnRegI64); + + RegI64 rhs = popI64(); + RegI64 srcDest = popI64ToSpecific(specific_.abiReturnRegI64); + + Label done; + + checkDivideByZero(rhs); + + if (callee == SymbolicAddress::DivI64) { + checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(false)); + } else if (callee == SymbolicAddress::ModI64) { + checkDivideSignedOverflow(rhs, srcDest, &done, ZeroOnOverflow(true)); + } + + masm.setupWasmABICall(); + masm.passABIArg(srcDest.high); + masm.passABIArg(srcDest.low); + masm.passABIArg(rhs.high); + masm.passABIArg(rhs.low); + CodeOffset raOffset = masm.callWithABI( + bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset())); + if (!createStackMap("emitDivOrModI64Bui[..]", raOffset)) { + return false; + } + + masm.bind(&done); + + freeI64(rhs); + pushI64(srcDest); + return true; +} +#endif // RABALDR_INT_DIV_I64_CALLOUT + +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT +bool BaseCompiler::emitConvertInt64ToFloatingCallout(SymbolicAddress callee, + ValType operandType, + ValType resultType) { + sync(); + + RegI64 input = popI64(); + + FunctionCall call{}; + + masm.setupWasmABICall(); +# ifdef JS_PUNBOX64 + MOZ_CRASH("BaseCompiler platform hook: emitConvertInt64ToFloatingCallout"); +# else + masm.passABIArg(input.high); + masm.passABIArg(input.low); +# endif + CodeOffset raOffset = masm.callWithABI( + bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset()), + resultType == ValType::F32 ? MoveOp::FLOAT32 : MoveOp::DOUBLE); + if (!createStackMap("emitConvertInt64To[..]", raOffset)) { + return false; + } + + freeI64(input); + + if (resultType == ValType::F32) { + pushF32(captureReturnedF32(call)); + } else { + pushF64(captureReturnedF64(call)); + } + + return true; +} +#endif // RABALDR_I64_TO_FLOAT_CALLOUT + +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT +// `Callee` always takes a double, so a float32 input must be converted. +bool BaseCompiler::emitConvertFloatingToInt64Callout(SymbolicAddress callee, + ValType operandType, + ValType resultType) { + RegF64 doubleInput; + if (operandType == ValType::F32) { + doubleInput = needF64(); + RegF32 input = popF32(); + masm.convertFloat32ToDouble(input, doubleInput); + freeF32(input); + } else { + doubleInput = popF64(); + } + + // We may need the value after the call for the ool check. + RegF64 otherReg = needF64(); + moveF64(doubleInput, otherReg); + pushF64(otherReg); + + sync(); + + FunctionCall call{}; + + masm.setupWasmABICall(); + masm.passABIArg(doubleInput, MoveOp::DOUBLE); + CodeOffset raOffset = masm.callWithABI( + bytecodeOffset(), callee, mozilla::Some(fr.getInstancePtrOffset())); + if (!createStackMap("emitConvertFloatin[..]", raOffset)) { + return false; + } + + freeF64(doubleInput); + + RegI64 rv = captureReturnedI64(); + + RegF64 inputVal = popF64(); + + TruncFlags flags = 0; + if (callee == SymbolicAddress::TruncateDoubleToUint64) { + flags |= TRUNC_UNSIGNED; + } + if (callee == SymbolicAddress::SaturatingTruncateDoubleToInt64 || + callee == SymbolicAddress::SaturatingTruncateDoubleToUint64) { + flags |= TRUNC_SATURATING; + } + + // If we're saturating, the callout will always produce the final result + // value. Otherwise, the callout value will return 0x8000000000000000 + // and we need to produce traps. + OutOfLineCode* ool = nullptr; + if (!(flags & TRUNC_SATURATING)) { + // The OOL check just succeeds or fails, it does not generate a value. + ool = addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64( + AnyReg(inputVal), rv, flags, bytecodeOffset())); + if (!ool) { + return false; + } + + masm.branch64(Assembler::Equal, rv, Imm64(0x8000000000000000), + ool->entry()); + masm.bind(ool->rejoin()); + } + + pushI64(rv); + freeF64(inputVal); + + return true; +} +#endif // RABALDR_FLOAT_TO_I64_CALLOUT + +bool BaseCompiler::emitGetLocal() { + uint32_t slot; + if (!iter_.readGetLocal(locals_, &slot)) { + return false; + } + + if (deadCode_) { + return true; + } + + // Local loads are pushed unresolved, ie, they may be deferred + // until needed, until they may be affected by a store, or until a + // sync. This is intended to reduce register pressure. + + switch (locals_[slot].kind()) { + case ValType::I32: + pushLocalI32(slot); + break; + case ValType::I64: + pushLocalI64(slot); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + pushLocalV128(slot); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + case ValType::F64: + pushLocalF64(slot); + break; + case ValType::F32: + pushLocalF32(slot); + break; + case ValType::Ref: + pushLocalRef(slot); + break; + } + + return true; +} + +template +bool BaseCompiler::emitSetOrTeeLocal(uint32_t slot) { + if (deadCode_) { + return true; + } + + bceLocalIsUpdated(slot); + switch (locals_[slot].kind()) { + case ValType::I32: { + RegI32 rv = popI32(); + syncLocal(slot); + fr.storeLocalI32(rv, localFromSlot(slot, MIRType::Int32)); + if (isSetLocal) { + freeI32(rv); + } else { + pushI32(rv); + } + break; + } + case ValType::I64: { + RegI64 rv = popI64(); + syncLocal(slot); + fr.storeLocalI64(rv, localFromSlot(slot, MIRType::Int64)); + if (isSetLocal) { + freeI64(rv); + } else { + pushI64(rv); + } + break; + } + case ValType::F64: { + RegF64 rv = popF64(); + syncLocal(slot); + fr.storeLocalF64(rv, localFromSlot(slot, MIRType::Double)); + if (isSetLocal) { + freeF64(rv); + } else { + pushF64(rv); + } + break; + } + case ValType::F32: { + RegF32 rv = popF32(); + syncLocal(slot); + fr.storeLocalF32(rv, localFromSlot(slot, MIRType::Float32)); + if (isSetLocal) { + freeF32(rv); + } else { + pushF32(rv); + } + break; + } + case ValType::V128: { +#ifdef ENABLE_WASM_SIMD + RegV128 rv = popV128(); + syncLocal(slot); + fr.storeLocalV128(rv, localFromSlot(slot, MIRType::Simd128)); + if (isSetLocal) { + freeV128(rv); + } else { + pushV128(rv); + } + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + } + case ValType::Ref: { + RegRef rv = popRef(); + syncLocal(slot); + fr.storeLocalRef(rv, localFromSlot(slot, MIRType::RefOrNull)); + if (isSetLocal) { + freeRef(rv); + } else { + pushRef(rv); + } + break; + } + } + + return true; +} + +bool BaseCompiler::emitSetLocal() { + uint32_t slot; + Nothing unused_value; + if (!iter_.readSetLocal(locals_, &slot, &unused_value)) { + return false; + } + return emitSetOrTeeLocal(slot); +} + +bool BaseCompiler::emitTeeLocal() { + uint32_t slot; + Nothing unused_value; + if (!iter_.readTeeLocal(locals_, &slot, &unused_value)) { + return false; + } + return emitSetOrTeeLocal(slot); +} + +bool BaseCompiler::emitGetGlobal() { + uint32_t id; + if (!iter_.readGetGlobal(&id)) { + return false; + } + + if (deadCode_) { + return true; + } + + const GlobalDesc& global = moduleEnv_.globals[id]; + + if (global.isConstant()) { + LitVal value = global.constantValue(); + switch (value.type().kind()) { + case ValType::I32: + pushI32(value.i32()); + break; + case ValType::I64: + pushI64(value.i64()); + break; + case ValType::F32: + pushF32(value.f32()); + break; + case ValType::F64: + pushF64(value.f64()); + break; + case ValType::Ref: + pushRef(intptr_t(value.ref().forCompiledCode())); + break; +#ifdef ENABLE_WASM_SIMD + case ValType::V128: + pushV128(value.v128()); + break; +#endif + default: + MOZ_CRASH("Global constant type"); + } + return true; + } + + switch (global.type().kind()) { + case ValType::I32: { + RegI32 rv = needI32(); + ScratchPtr tmp(*this); + masm.load32(addressOfGlobalVar(global, tmp), rv); + pushI32(rv); + break; + } + case ValType::I64: { + RegI64 rv = needI64(); + ScratchPtr tmp(*this); + masm.load64(addressOfGlobalVar(global, tmp), rv); + pushI64(rv); + break; + } + case ValType::F32: { + RegF32 rv = needF32(); + ScratchPtr tmp(*this); + masm.loadFloat32(addressOfGlobalVar(global, tmp), rv); + pushF32(rv); + break; + } + case ValType::F64: { + RegF64 rv = needF64(); + ScratchPtr tmp(*this); + masm.loadDouble(addressOfGlobalVar(global, tmp), rv); + pushF64(rv); + break; + } + case ValType::Ref: { + RegRef rv = needRef(); + ScratchPtr tmp(*this); + masm.loadPtr(addressOfGlobalVar(global, tmp), rv); + pushRef(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 rv = needV128(); + ScratchPtr tmp(*this); + masm.loadUnalignedSimd128(addressOfGlobalVar(global, tmp), rv); + pushV128(rv); + break; + } +#endif + default: + MOZ_CRASH("Global variable type"); + break; + } + return true; +} + +bool BaseCompiler::emitSetGlobal() { + uint32_t id; + Nothing unused_value; + if (!iter_.readSetGlobal(&id, &unused_value)) { + return false; + } + + if (deadCode_) { + return true; + } + + const GlobalDesc& global = moduleEnv_.globals[id]; + + switch (global.type().kind()) { + case ValType::I32: { + RegI32 rv = popI32(); + ScratchPtr tmp(*this); + masm.store32(rv, addressOfGlobalVar(global, tmp)); + freeI32(rv); + break; + } + case ValType::I64: { + RegI64 rv = popI64(); + ScratchPtr tmp(*this); + masm.store64(rv, addressOfGlobalVar(global, tmp)); + freeI64(rv); + break; + } + case ValType::F32: { + RegF32 rv = popF32(); + ScratchPtr tmp(*this); + masm.storeFloat32(rv, addressOfGlobalVar(global, tmp)); + freeF32(rv); + break; + } + case ValType::F64: { + RegF64 rv = popF64(); + ScratchPtr tmp(*this); + masm.storeDouble(rv, addressOfGlobalVar(global, tmp)); + freeF64(rv); + break; + } + case ValType::Ref: { + RegPtr valueAddr(PreBarrierReg); + needPtr(valueAddr); + { + ScratchPtr tmp(*this); + masm.computeEffectiveAddress(addressOfGlobalVar(global, tmp), + valueAddr); + } + RegRef rv = popRef(); + // emitBarrieredStore preserves rv + if (!emitBarrieredStore(Nothing(), valueAddr, rv, PreBarrierKind::Normal, + PostBarrierKind::Imprecise)) { + return false; + } + freeRef(rv); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 rv = popV128(); + ScratchPtr tmp(*this); + masm.storeUnalignedSimd128(rv, addressOfGlobalVar(global, tmp)); + freeV128(rv); + break; + } +#endif + default: + MOZ_CRASH("Global variable type"); + break; + } + return true; +} + +bool BaseCompiler::emitLoad(ValType type, Scalar::Type viewType) { + LinearMemoryAddress addr; + if (!iter_.readLoad(type, Scalar::byteSize(viewType), &addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadCommon(&access, AccessCheck(), type); + return true; +} + +bool BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType) { + LinearMemoryAddress addr; + Nothing unused_value; + if (!iter_.readStore(resultType, Scalar::byteSize(viewType), &addr, + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + storeCommon(&access, AccessCheck(), resultType); + return true; +} + +bool BaseCompiler::emitSelect(bool typed) { + StackType type; + Nothing unused_trueValue; + Nothing unused_falseValue; + Nothing unused_condition; + if (!iter_.readSelect(typed, &type, &unused_trueValue, &unused_falseValue, + &unused_condition)) { + return false; + } + + if (deadCode_) { + resetLatentOp(); + return true; + } + + // I32 condition on top, then false, then true. + + Label done; + BranchState b(&done); + emitBranchSetup(&b); + + switch (type.valType().kind()) { + case ValType::I32: { + RegI32 r, rs; + pop2xI32(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveI32(rs, r); + masm.bind(&done); + freeI32(rs); + pushI32(r); + break; + } + case ValType::I64: { +#ifdef JS_CODEGEN_X86 + // There may be as many as four Int64 values in registers at a time: two + // for the latent branch operands, and two for the true/false values we + // normally pop before executing the branch. On x86 this is one value + // too many, so we need to generate more complicated code here, and for + // simplicity's sake we do so even if the branch operands are not Int64. + // However, the resulting control flow diamond is complicated since the + // arms of the diamond will have to stay synchronized with respect to + // their evaluation stack and regalloc state. To simplify further, we + // use a double branch and a temporary boolean value for now. + RegI32 temp = needI32(); + moveImm32(0, temp); + if (!emitBranchPerform(&b)) { + return false; + } + moveImm32(1, temp); + masm.bind(&done); + + Label trueValue; + RegI64 r, rs; + pop2xI64(&r, &rs); + masm.branch32(Assembler::Equal, temp, Imm32(0), &trueValue); + moveI64(rs, r); + masm.bind(&trueValue); + freeI32(temp); + freeI64(rs); + pushI64(r); +#else + RegI64 r, rs; + pop2xI64(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveI64(rs, r); + masm.bind(&done); + freeI64(rs); + pushI64(r); +#endif + break; + } + case ValType::F32: { + RegF32 r, rs; + pop2xF32(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveF32(rs, r); + masm.bind(&done); + freeF32(rs); + pushF32(r); + break; + } + case ValType::F64: { + RegF64 r, rs; + pop2xF64(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveF64(rs, r); + masm.bind(&done); + freeF64(rs); + pushF64(r); + break; + } +#ifdef ENABLE_WASM_SIMD + case ValType::V128: { + RegV128 r, rs; + pop2xV128(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveV128(rs, r); + masm.bind(&done); + freeV128(rs); + pushV128(r); + break; + } +#endif + case ValType::Ref: { + RegRef r, rs; + pop2xRef(&r, &rs); + if (!emitBranchPerform(&b)) { + return false; + } + moveRef(rs, r); + masm.bind(&done); + freeRef(rs); + pushRef(r); + break; + } + default: { + MOZ_CRASH("select type"); + } + } + + return true; +} + +void BaseCompiler::emitCompareI32(Assembler::Condition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::I32); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + int32_t c; + if (popConst(&c)) { + RegI32 r = popI32(); + masm.cmp32Set(compareOp, r, Imm32(c), r); + pushI32(r); + } else { + RegI32 r, rs; + pop2xI32(&r, &rs); + masm.cmp32Set(compareOp, r, rs, r); + freeI32(rs); + pushI32(r); + } +} + +void BaseCompiler::emitCompareI64(Assembler::Condition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::I64); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + RegI64 rs0, rs1; + pop2xI64(&rs0, &rs1); + RegI32 rd(fromI64(rs0)); + cmp64Set(compareOp, rs0, rs1, rd); + freeI64(rs1); + freeI64Except(rs0, rd); + pushI32(rd); +} + +void BaseCompiler::emitCompareF32(Assembler::DoubleCondition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::F32); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + Label across; + RegF32 rs0, rs1; + pop2xF32(&rs0, &rs1); + RegI32 rd = needI32(); + moveImm32(1, rd); + masm.branchFloat(compareOp, rs0, rs1, &across); + moveImm32(0, rd); + masm.bind(&across); + freeF32(rs0); + freeF32(rs1); + pushI32(rd); +} + +void BaseCompiler::emitCompareF64(Assembler::DoubleCondition compareOp, + ValType compareType) { + MOZ_ASSERT(compareType == ValType::F64); + + if (sniffConditionalControlCmp(compareOp, compareType)) { + return; + } + + Label across; + RegF64 rs0, rs1; + pop2xF64(&rs0, &rs1); + RegI32 rd = needI32(); + moveImm32(1, rd); + masm.branchDouble(compareOp, rs0, rs1, &across); + moveImm32(0, rd); + masm.bind(&across); + freeF64(rs0); + freeF64(rs1); + pushI32(rd); +} + +void BaseCompiler::emitCompareRef(Assembler::Condition compareOp, + ValType compareType) { + MOZ_ASSERT(!sniffConditionalControlCmp(compareOp, compareType)); + + RegRef rs1, rs2; + pop2xRef(&rs1, &rs2); + RegI32 rd = needI32(); + masm.cmpPtrSet(compareOp, rs1, rs2, rd); + freeRef(rs1); + freeRef(rs2); + pushI32(rd); +} + +bool BaseCompiler::emitInstanceCall(const SymbolicAddressSignature& builtin) { + // See declaration (WasmBCClass.h) for info on the relationship between the + // compiler's value stack and the argument order for the to-be-called + // function. + const MIRType* argTypes = builtin.argTypes; + MOZ_ASSERT(argTypes[0] == MIRType::Pointer); + + sync(); + + uint32_t numNonInstanceArgs = builtin.numArgs - 1 /* instance */; + size_t stackSpace = stackConsumed(numNonInstanceArgs); + + FunctionCall baselineCall{}; + beginCall(baselineCall, UseABI::System, RestoreRegisterStateAndRealm::True); + + ABIArg instanceArg = reservePointerArgument(&baselineCall); + + startCallArgs(StackArgAreaSizeUnaligned(builtin), &baselineCall); + for (uint32_t i = 1; i < builtin.numArgs; i++) { + ValType t; + switch (argTypes[i]) { + case MIRType::Int32: + t = ValType::I32; + break; + case MIRType::Int64: + t = ValType::I64; + break; + case MIRType::Float32: + t = ValType::F32; + break; + case MIRType::RefOrNull: + t = RefType::extern_(); + break; + case MIRType::Pointer: + // Instance function args can now be uninterpreted pointers (eg, for + // the cases PostBarrier and PostBarrierFilter) so we simply treat + // them like the equivalently sized integer. + t = ValType::fromMIRType(TargetWordMIRType()); + break; + default: + MOZ_CRASH("Unexpected type"); + } + passArg(t, peek(numNonInstanceArgs - i), &baselineCall); + } + CodeOffset raOffset = + builtinInstanceMethodCall(builtin, instanceArg, baselineCall); + if (!createStackMap("emitInstanceCall", raOffset)) { + return false; + } + + endCall(baselineCall, stackSpace); + + popValueStackBy(numNonInstanceArgs); + + // Note, many clients of emitInstanceCall currently assume that pushing the + // result here does not destroy ReturnReg. + // + // Furthermore, clients assume that if builtin.retType != MIRType::None, the + // callee will have returned a result and left it in ReturnReg for us to + // find, and that that register will not be destroyed here (or above). + + // For the return type only, MIRType::None is used to indicate that the + // call doesn't return a result, that is, returns a C/C++ "void". + + if (builtin.retType != MIRType::None) { + pushReturnValueOfCall(baselineCall, builtin.retType); + } + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Reference types. + +bool BaseCompiler::emitRefFunc() { + return emitInstanceCallOp(SASigRefFunc, + [this](uint32_t* funcIndex) -> bool { + return iter_.readRefFunc(funcIndex); + }); +} + +bool BaseCompiler::emitRefNull() { + RefType type; + if (!iter_.readRefNull(&type)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushRef(NULLREF_VALUE); + return true; +} + +bool BaseCompiler::emitRefIsNull() { + Nothing nothing; + if (!iter_.readRefIsNull(¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegRef r = popRef(); + RegI32 rd = narrowRef(r); + + masm.cmpPtrSet(Assembler::Equal, r, ImmWord(NULLREF_VALUE), rd); + pushI32(rd); + return true; +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +bool BaseCompiler::emitRefAsNonNull() { + Nothing nothing; + if (!iter_.readRefAsNonNull(¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegRef rp = popRef(); + Label ok; + masm.branchTestPtr(Assembler::NonZero, rp, rp, &ok); + trap(Trap::NullPointerDereference); + masm.bind(&ok); + pushRef(rp); + + return true; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic operations. + +bool BaseCompiler::emitAtomicCmpXchg(ValType type, Scalar::Type viewType) { + LinearMemoryAddress addr; + Nothing unused{}; + if (!iter_.readAtomicCmpXchg(&addr, type, Scalar::byteSize(viewType), &unused, + &unused)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Full()); + atomicCmpXchg(&access, type); + return true; +} + +bool BaseCompiler::emitAtomicLoad(ValType type, Scalar::Type viewType) { + LinearMemoryAddress addr; + if (!iter_.readAtomicLoad(&addr, type, Scalar::byteSize(viewType))) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Load()); + atomicLoad(&access, type); + return true; +} + +bool BaseCompiler::emitAtomicRMW(ValType type, Scalar::Type viewType, + AtomicOp op) { + LinearMemoryAddress addr; + Nothing unused_value; + if (!iter_.readAtomicRMW(&addr, type, Scalar::byteSize(viewType), + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Full()); + atomicRMW(&access, type, op); + return true; +} + +bool BaseCompiler::emitAtomicStore(ValType type, Scalar::Type viewType) { + LinearMemoryAddress addr; + Nothing unused_value; + if (!iter_.readAtomicStore(&addr, type, Scalar::byteSize(viewType), + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Store()); + atomicStore(&access, type); + return true; +} + +bool BaseCompiler::emitAtomicXchg(ValType type, Scalar::Type viewType) { + LinearMemoryAddress addr; + Nothing unused_value; + if (!iter_.readAtomicRMW(&addr, type, Scalar::byteSize(viewType), + &unused_value)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset(), + Synchronization::Full()); + atomicXchg(&access, type); + return true; +} + +bool BaseCompiler::emitWait(ValType type, uint32_t byteSize) { + Nothing nothing; + LinearMemoryAddress addr; + if (!iter_.readWait(&addr, type, byteSize, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access( + type.kind() == ValType::I32 ? Scalar::Int32 : Scalar::Int64, addr.align, + addr.offset, bytecodeOffset()); + return atomicWait(type, &access); +} + +bool BaseCompiler::emitWake() { + Nothing nothing; + LinearMemoryAddress addr; + if (!iter_.readWake(&addr, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(Scalar::Int32, addr.align, addr.offset, + bytecodeOffset()); + return atomicWake(&access); +} + +bool BaseCompiler::emitFence() { + if (!iter_.readFence()) { + return false; + } + if (deadCode_) { + return true; + } + masm.memoryBarrier(MembarFull); + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk memory operations. + +bool BaseCompiler::emitMemoryGrow() { + return emitInstanceCallOp( + !usesMemory() || isMem32() ? SASigMemoryGrowM32 : SASigMemoryGrowM64, + [this]() -> bool { + Nothing arg; + return iter_.readMemoryGrow(&arg); + }); +} + +bool BaseCompiler::emitMemorySize() { + return emitInstanceCallOp( + !usesMemory() || isMem32() ? SASigMemorySizeM32 : SASigMemorySizeM64, + [this]() -> bool { return iter_.readMemorySize(); }); +} + +bool BaseCompiler::emitMemCopy() { + uint32_t dstMemOrTableIndex = 0; + uint32_t srcMemOrTableIndex = 0; + Nothing nothing; + if (!iter_.readMemOrTableCopy(true, &dstMemOrTableIndex, ¬hing, + &srcMemOrTableIndex, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + + if (isMem32()) { + int32_t signedLength; + if (peekConst(&signedLength) && signedLength != 0 && + uint32_t(signedLength) <= MaxInlineMemoryCopyLength) { + memCopyInlineM32(); + return true; + } + } + + return memCopyCall(); +} + +bool BaseCompiler::memCopyCall() { + pushHeapBase(); + return emitInstanceCall( + usesSharedMemory() + ? (isMem32() ? SASigMemCopySharedM32 : SASigMemCopySharedM64) + : (isMem32() ? SASigMemCopyM32 : SASigMemCopyM64)); +} + +bool BaseCompiler::emitMemFill() { + Nothing nothing; + if (!iter_.readMemFill(¬hing, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + + if (isMem32()) { + int32_t signedLength; + int32_t signedValue; + if (peek2xConst(&signedLength, &signedValue) && signedLength != 0 && + uint32_t(signedLength) <= MaxInlineMemoryFillLength) { + memFillInlineM32(); + return true; + } + } + return memFillCall(); +} + +bool BaseCompiler::memFillCall() { + pushHeapBase(); + return emitInstanceCall( + usesSharedMemory() + ? (isMem32() ? SASigMemFillSharedM32 : SASigMemFillSharedM64) + : (isMem32() ? SASigMemFillM32 : SASigMemFillM64)); +} + +bool BaseCompiler::emitMemInit() { + return emitInstanceCallOp( + (!usesMemory() || isMem32() ? SASigMemInitM32 : SASigMemInitM64), + [this](uint32_t* segIndex) -> bool { + Nothing nothing; + if (iter_.readMemOrTableInit(/*isMem*/ true, segIndex, nullptr, + ¬hing, ¬hing, ¬hing)) { + return true; + } + return false; + }); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk table operations. + +bool BaseCompiler::emitTableCopy() { + uint32_t dstMemOrTableIndex = 0; + uint32_t srcMemOrTableIndex = 0; + Nothing nothing; + if (!iter_.readMemOrTableCopy(false, &dstMemOrTableIndex, ¬hing, + &srcMemOrTableIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushI32(dstMemOrTableIndex); + pushI32(srcMemOrTableIndex); + return emitInstanceCall(SASigTableCopy); +} + +bool BaseCompiler::emitTableInit() { + return emitInstanceCallOp( + SASigTableInit, + [this](uint32_t* segIndex, uint32_t* dstTableIndex) -> bool { + Nothing nothing; + return iter_.readMemOrTableInit(/*isMem*/ false, segIndex, + dstTableIndex, ¬hing, ¬hing, + ¬hing); + }); +} + +bool BaseCompiler::emitTableFill() { + // fill(start:u32, val:ref, len:u32, table:u32) -> void + return emitInstanceCallOp( + SASigTableFill, [this](uint32_t* tableIndex) -> bool { + Nothing nothing; + return iter_.readTableFill(tableIndex, ¬hing, ¬hing, ¬hing); + }); +} + +bool BaseCompiler::emitMemDiscard() { + Nothing nothing; + if (!iter_.readMemDiscard(¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + + pushHeapBase(); + return emitInstanceCall( + usesSharedMemory() + ? (isMem32() ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64) + : (isMem32() ? SASigMemDiscardM32 : SASigMemDiscardM64)); +} + +bool BaseCompiler::emitTableGet() { + uint32_t tableIndex; + Nothing nothing; + if (!iter_.readTableGet(&tableIndex, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + if (moduleEnv_.tables[tableIndex].elemType.tableRepr() == TableRepr::Ref) { + return emitTableGetAnyRef(tableIndex); + } + pushI32(tableIndex); + // get(index:u32, table:u32) -> AnyRef + return emitInstanceCall(SASigTableGet); +} + +bool BaseCompiler::emitTableGrow() { + // grow(initValue:anyref, delta:u32, table:u32) -> u32 + return emitInstanceCallOp( + SASigTableGrow, [this](uint32_t* tableIndex) -> bool { + Nothing nothing; + return iter_.readTableGrow(tableIndex, ¬hing, ¬hing); + }); +} + +bool BaseCompiler::emitTableSet() { + uint32_t tableIndex; + Nothing nothing; + if (!iter_.readTableSet(&tableIndex, ¬hing, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + if (moduleEnv_.tables[tableIndex].elemType.tableRepr() == TableRepr::Ref) { + return emitTableSetAnyRef(tableIndex); + } + pushI32(tableIndex); + // set(index:u32, value:ref, table:u32) -> void + return emitInstanceCall(SASigTableSet); +} + +bool BaseCompiler::emitTableSize() { + uint32_t tableIndex; + if (!iter_.readTableSize(&tableIndex)) { + return false; + } + if (deadCode_) { + return true; + } + + RegPtr instance = needPtr(); + RegI32 length = needI32(); + + fr.loadInstancePtr(instance); + loadTableLength(tableIndex, instance, length); + + pushI32(length); + freePtr(instance); + return true; +} + +void BaseCompiler::emitTableBoundsCheck(uint32_t tableIndex, RegI32 index, + RegPtr instance) { + Label ok; + masm.wasmBoundsCheck32( + Assembler::Condition::Below, index, + addressOfTableField(tableIndex, offsetof(TableInstanceData, length), + instance), + &ok); + masm.wasmTrap(wasm::Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&ok); +} + +bool BaseCompiler::emitTableGetAnyRef(uint32_t tableIndex) { + RegPtr instance = needPtr(); + RegPtr elements = needPtr(); + RegI32 index = popI32(); + + fr.loadInstancePtr(instance); + emitTableBoundsCheck(tableIndex, index, instance); + loadTableElements(tableIndex, instance, elements); + masm.loadPtr(BaseIndex(elements, index, ScalePointer), elements); + + pushRef(RegRef(elements)); + freeI32(index); + freePtr(instance); + + return true; +} + +bool BaseCompiler::emitTableSetAnyRef(uint32_t tableIndex) { + // Create temporaries for valueAddr that is not in the prebarrier register + // and can be consumed by the barrier operation + RegPtr valueAddr = RegPtr(PreBarrierReg); + needPtr(valueAddr); + + RegPtr instance = needPtr(); + RegPtr elements = needPtr(); + RegRef value = popRef(); + RegI32 index = popI32(); + + // x86 is one register too short for this operation, shuffle `value` back + // onto the stack until it is needed. +#ifdef JS_CODEGEN_X86 + pushRef(value); +#endif + + fr.loadInstancePtr(instance); + emitTableBoundsCheck(tableIndex, index, instance); + loadTableElements(tableIndex, instance, elements); + masm.computeEffectiveAddress(BaseIndex(elements, index, ScalePointer), + valueAddr); + + freeI32(index); + freePtr(elements); + freePtr(instance); + +#ifdef JS_CODEGEN_X86 + value = popRef(); +#endif + + if (!emitBarrieredStore(Nothing(), valueAddr, value, PreBarrierKind::Normal, + PostBarrierKind::Precise)) { + return false; + } + freeRef(value); + return true; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Data and element segment management. + +bool BaseCompiler::emitDataOrElemDrop(bool isData) { + return emitInstanceCallOp( + isData ? SASigDataDrop : SASigElemDrop, [&](uint32_t* segIndex) -> bool { + return iter_.readDataOrElemDrop(isData, segIndex); + }); +} + +////////////////////////////////////////////////////////////////////////////// +// +// General object support. + +void BaseCompiler::emitPreBarrier(RegPtr valueAddr) { + Label skipBarrier; + ScratchPtr scratch(*this); + +#ifdef RABALDR_PIN_INSTANCE + Register instance(InstanceReg); +#else + Register instance(scratch); + fr.loadInstancePtr(instance); +#endif + + EmitWasmPreBarrierGuard(masm, instance, scratch, valueAddr, + /*valueOffset=*/0, &skipBarrier, nullptr); + +#ifndef RABALDR_PIN_INSTANCE + fr.loadInstancePtr(instance); +#endif +#ifdef JS_CODEGEN_ARM64 + // The prebarrier stub assumes the PseudoStackPointer is set up. It is OK + // to just move the sp to x28 here because x28 is not being used by the + // baseline compiler and need not be saved or restored. + MOZ_ASSERT(!GeneralRegisterSet::All().hasRegisterIndex(x28.asUnsized())); + masm.Mov(x28, sp); +#endif + // The prebarrier call preserves all volatile registers + EmitWasmPreBarrierCall(masm, instance, scratch, valueAddr, /*valueOffset=*/0); + + masm.bind(&skipBarrier); +} + +bool BaseCompiler::emitPostBarrierImprecise(const Maybe& object, + RegPtr valueAddr, RegRef value) { + // We must force a sync before the guard so that locals are in a consistent + // location for whether or not the post-barrier call is taken. + sync(); + + // Emit a guard to skip the post-barrier call if it is not needed. + Label skipBarrier; + RegPtr otherScratch = needPtr(); + EmitWasmPostBarrierGuard(masm, object, otherScratch, value, &skipBarrier); + freePtr(otherScratch); + + // Push `object` and `value` to preserve them across the call. + if (object) { + pushRef(*object); + } + pushRef(value); + + // The `valueAddr` is a raw pointer to the cell within some GC object or + // instance area, and we are careful so that the GC will not run while the + // post-barrier call is active, so push a uintptr_t value. + pushPtr(valueAddr); + if (!emitInstanceCall(SASigPostBarrier)) { + return false; + } + + // Restore `object` and `value`. + popRef(value); + if (object) { + popRef(*object); + } + + masm.bind(&skipBarrier); + return true; +} + +bool BaseCompiler::emitPostBarrierPrecise(const Maybe& object, + RegPtr valueAddr, RegRef prevValue, + RegRef value) { + // Push `object` and `value` to preserve them across the call. + if (object) { + pushRef(*object); + } + pushRef(value); + + // Push the arguments and call the precise post-barrier + pushPtr(valueAddr); + pushRef(prevValue); + if (!emitInstanceCall(SASigPostBarrierPrecise)) { + return false; + } + + // Restore `object` and `value`. + popRef(value); + if (object) { + popRef(*object); + } + + return true; +} + +bool BaseCompiler::emitBarrieredStore(const Maybe& object, + RegPtr valueAddr, RegRef value, + PreBarrierKind preBarrierKind, + PostBarrierKind postBarrierKind) { + // TODO/AnyRef-boxing: With boxed immediates and strings, the write + // barrier is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + + // The pre-barrier preserves all allocated registers. + if (preBarrierKind == PreBarrierKind::Normal) { + emitPreBarrier(valueAddr); + } + + // The precise post-barrier requires the previous value stored in the field, + // in order to know if the previous store buffer entry needs to be removed. + RegRef prevValue; + if (postBarrierKind == PostBarrierKind::Precise) { + prevValue = needRef(); + masm.loadPtr(Address(valueAddr, 0), prevValue); + } + + // Store the value + masm.storePtr(value, Address(valueAddr, 0)); + + // The post-barrier preserves object and value. + if (postBarrierKind == PostBarrierKind::Precise) { + return emitPostBarrierPrecise(object, valueAddr, prevValue, value); + } + return emitPostBarrierImprecise(object, valueAddr, value); +} + +void BaseCompiler::emitBarrieredClear(RegPtr valueAddr) { + // TODO/AnyRef-boxing: With boxed immediates and strings, the write + // barrier is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + + // The pre-barrier preserves all allocated registers. + emitPreBarrier(valueAddr); + + // Store null + masm.storePtr(ImmWord(0), Address(valueAddr, 0)); + + // No post-barrier is needed, as null does not require a store buffer entry +} + +////////////////////////////////////////////////////////////////////////////// +// +// GC proposal. + +#ifdef ENABLE_WASM_GC + +RegPtr BaseCompiler::loadTypeDefInstanceData(uint32_t typeIndex) { + RegPtr rp = needPtr(); + RegPtr instance; +# ifndef RABALDR_PIN_INSTANCE + instance = rp; + fr.loadInstancePtr(instance); +# else + // We can use the pinned instance register. + instance = RegPtr(InstanceReg); +# endif + masm.computeEffectiveAddress( + Address(instance, Instance::offsetInData( + moduleEnv_.offsetOfTypeDefInstanceData(typeIndex))), + rp); + return rp; +} + +RegPtr BaseCompiler::loadSuperTypeVector(uint32_t typeIndex) { + RegPtr rp = needPtr(); + RegPtr instance; +# ifndef RABALDR_PIN_INSTANCE + // We need to load the instance register, but can use the destination + // register as a temporary. + instance = rp; + fr.loadInstancePtr(rp); +# else + // We can use the pinned instance register. + instance = RegPtr(InstanceReg); +# endif + masm.loadPtr( + Address(instance, Instance::offsetInData( + moduleEnv_.offsetOfSuperTypeVector(typeIndex))), + rp); + return rp; +} + +/* static */ +void BaseCompiler::SignalNullCheck::emitNullCheck(BaseCompiler* bc, RegRef rp) { + Label ok; + MacroAssembler& masm = bc->masm; + masm.branchTestPtr(Assembler::NonZero, rp, rp, &ok); + bc->trap(Trap::NullPointerDereference); + masm.bind(&ok); +} + +/* static */ +void BaseCompiler::SignalNullCheck::emitTrapSite(BaseCompiler* bc) { + wasm::BytecodeOffset trapOffset(bc->bytecodeOffset()); + MacroAssembler& masm = bc->masm; + masm.append(wasm::Trap::NullPointerDereference, + wasm::TrapSite(masm.currentOffset(), trapOffset)); +} + +RegPtr BaseCompiler::emitGcArrayGetData(RegRef rp) { + // `rp` points at a WasmArrayObject. Return a reg holding the value of its + // `data_` field. + RegPtr rdata = needPtr(); + masm.loadPtr(Address(rp, WasmArrayObject::offsetOfData()), rdata); + return rdata; +} + +template +RegI32 BaseCompiler::emitGcArrayGetNumElements(RegRef rp) { + // `rp` points at a WasmArrayObject. Return a reg holding the value of its + // `numElements_` field. + STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; + RegI32 numElements = needI32(); + NullCheckPolicy::emitTrapSite(this); + masm.load32(Address(rp, WasmArrayObject::offsetOfNumElements()), numElements); + return numElements; +} + +void BaseCompiler::emitGcArrayBoundsCheck(RegI32 index, RegI32 numElements) { + Label inBounds; + masm.branch32(Assembler::Below, index, numElements, &inBounds); + masm.wasmTrap(Trap::OutOfBounds, bytecodeOffset()); + masm.bind(&inBounds); +} + +template +void BaseCompiler::emitGcGet(FieldType type, FieldWideningOp wideningOp, + const T& src) { + switch (type.kind()) { + case FieldType::I8: { + MOZ_ASSERT(wideningOp != FieldWideningOp::None); + RegI32 r = needI32(); + NullCheckPolicy::emitTrapSite(this); + if (wideningOp == FieldWideningOp::Unsigned) { + masm.load8ZeroExtend(src, r); + } else { + masm.load8SignExtend(src, r); + } + pushI32(r); + break; + } + case FieldType::I16: { + MOZ_ASSERT(wideningOp != FieldWideningOp::None); + RegI32 r = needI32(); + NullCheckPolicy::emitTrapSite(this); + if (wideningOp == FieldWideningOp::Unsigned) { + masm.load16ZeroExtend(src, r); + } else { + masm.load16SignExtend(src, r); + } + pushI32(r); + break; + } + case FieldType::I32: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegI32 r = needI32(); + NullCheckPolicy::emitTrapSite(this); + masm.load32(src, r); + pushI32(r); + break; + } + case FieldType::I64: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegI64 r = needI64(); + NullCheckPolicy::emitTrapSite(this); + masm.load64(src, r); + pushI64(r); + break; + } + case FieldType::F32: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegF32 r = needF32(); + NullCheckPolicy::emitTrapSite(this); + masm.loadFloat32(src, r); + pushF32(r); + break; + } + case FieldType::F64: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegF64 r = needF64(); + NullCheckPolicy::emitTrapSite(this); + masm.loadDouble(src, r); + pushF64(r); + break; + } +# ifdef ENABLE_WASM_SIMD + case FieldType::V128: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegV128 r = needV128(); + NullCheckPolicy::emitTrapSite(this); + masm.loadUnalignedSimd128(src, r); + pushV128(r); + break; + } +# endif + case FieldType::Ref: { + MOZ_ASSERT(wideningOp == FieldWideningOp::None); + RegRef r = needRef(); + NullCheckPolicy::emitTrapSite(this); + masm.loadPtr(src, r); + pushRef(r); + break; + } + default: { + MOZ_CRASH("Unexpected field type"); + } + } +} + +template +void BaseCompiler::emitGcSetScalar(const T& dst, FieldType type, AnyReg value) { + NullCheckPolicy::emitTrapSite(this); + switch (type.kind()) { + case FieldType::I8: { + masm.store8(value.i32(), dst); + break; + } + case FieldType::I16: { + masm.store16(value.i32(), dst); + break; + } + case FieldType::I32: { + masm.store32(value.i32(), dst); + break; + } + case FieldType::I64: { + masm.store64(value.i64(), dst); + break; + } + case FieldType::F32: { + masm.storeFloat32(value.f32(), dst); + break; + } + case FieldType::F64: { + masm.storeDouble(value.f64(), dst); + break; + } +# ifdef ENABLE_WASM_SIMD + case FieldType::V128: { + masm.storeUnalignedSimd128(value.v128(), dst); + break; + } +# endif + default: { + MOZ_CRASH("Unexpected field type"); + } + } +} + +template +bool BaseCompiler::emitGcStructSet(RegRef object, RegPtr areaBase, + uint32_t areaOffset, FieldType fieldType, + AnyReg value, + PreBarrierKind preBarrierKind) { + // Easy path if the field is a scalar + if (!fieldType.isRefRepr()) { + emitGcSetScalar(Address(areaBase, areaOffset), + fieldType, value); + freeAny(value); + return true; + } + + // Create temporary for the valueAddr that is not in the prebarrier register + // and can be consumed by the barrier operation + RegPtr valueAddr = RegPtr(PreBarrierReg); + needPtr(valueAddr); + masm.computeEffectiveAddress(Address(areaBase, areaOffset), valueAddr); + + NullCheckPolicy::emitNullCheck(this, object); + + // emitBarrieredStore preserves object and value + if (!emitBarrieredStore(Some(object), valueAddr, value.ref(), preBarrierKind, + PostBarrierKind::Imprecise)) { + return false; + } + freeRef(value.ref()); + + return true; +} + +bool BaseCompiler::emitGcArraySet(RegRef object, RegPtr data, RegI32 index, + const ArrayType& arrayType, AnyReg value, + PreBarrierKind preBarrierKind) { + // Try to use a base index store instruction if the field type fits in a + // shift immediate. If not we shift the index manually and then unshift + // it after the store. We don't use an extra register for this because we + // don't have any to spare on x86. + uint32_t shift = arrayType.elementType_.indexingShift(); + Scale scale; + bool shiftedIndex = false; + if (IsShiftInScaleRange(shift)) { + scale = ShiftToScale(shift); + } else { + masm.lshiftPtr(Imm32(shift), index); + scale = TimesOne; + shiftedIndex = true; + } + auto unshiftIndex = mozilla::MakeScopeExit([&] { + if (shiftedIndex) { + masm.rshiftPtr(Imm32(shift), index); + } + }); + + // Easy path if the field is a scalar + if (!arrayType.elementType_.isRefRepr()) { + emitGcSetScalar(BaseIndex(data, index, scale, 0), + arrayType.elementType_, value); + return true; + } + + // Create temporaries for valueAddr that is not in the prebarrier register + // and can be consumed by the barrier operation + RegPtr valueAddr = RegPtr(PreBarrierReg); + needPtr(valueAddr); + masm.computeEffectiveAddress(BaseIndex(data, index, scale, 0), valueAddr); + + // Save state for after barriered write + pushPtr(data); + pushI32(index); + + // emitBarrieredStore preserves object and value + if (!emitBarrieredStore(Some(object), valueAddr, value.ref(), preBarrierKind, + PostBarrierKind::Imprecise)) { + return false; + } + + // Restore state + popI32(index); + popPtr(data); + + return true; +} + +bool BaseCompiler::emitStructNew() { + uint32_t typeIndex; + BaseNothingVector args{}; + if (!iter_.readStructNew(&typeIndex, &args)) { + return false; + } + + if (deadCode_) { + return true; + } + + const StructType& structType = (*moduleEnv_.types)[typeIndex].structType(); + + // Allocate an uninitialized struct. This requires the type definition + // for the struct to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + if (!emitInstanceCall(SASigStructNewUninit)) { + return false; + } + + // Optimization opportunity: Iterate backward to pop arguments off the + // stack. This will generate more instructions than we want, since we + // really only need to pop the stack once at the end, not for every element, + // but to do better we need a bit more machinery to load elements off the + // stack into registers. + + bool isOutlineStruct = structType.size_ > WasmStructObject_MaxInlineBytes; + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + needPtr(RegPtr(PreBarrierReg)); + + RegRef object = popRef(); + RegPtr outlineBase = isOutlineStruct ? needPtr() : RegPtr(); + + // Free the barrier reg after we've allocated all registers + freePtr(RegPtr(PreBarrierReg)); + + // Optimization opportunity: when the value being stored is a known + // zero/null we need store nothing. This case may be somewhat common + // because struct.new forces a value to be specified for every field. + + // Optimization opportunity: this loop reestablishes the outline base pointer + // every iteration, which really isn't very clever. It would be better to + // establish it once before we start, then re-set it if/when we transition + // from the out-of-line area back to the in-line area. That would however + // require making ::emitGcStructSet preserve that register, which it + // currently doesn't. + + uint32_t fieldIndex = structType.fields_.length(); + while (fieldIndex-- > 0) { + const StructField& field = structType.fields_[fieldIndex]; + FieldType fieldType = field.type; + uint32_t fieldOffset = field.offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + // Reserve the barrier reg if we might need it for this store + if (fieldType.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + AnyReg value = popAny(); + // Free the barrier reg now that we've loaded the value + if (fieldType.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + if (areaIsOutline) { + // Load the outline data pointer + masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()), + outlineBase); + + // Consumes value and outline data, object is preserved by this call. + if (!emitGcStructSet(object, outlineBase, areaOffset, + fieldType, value, + PreBarrierKind::None)) { + return false; + } + } else { + // Consumes value. object is unchanged by this call. + if (!emitGcStructSet( + object, RegPtr(object), + WasmStructObject::offsetOfInlineData() + areaOffset, fieldType, + value, PreBarrierKind::None)) { + return false; + } + } + } + + if (isOutlineStruct) { + freePtr(outlineBase); + } + pushRef(object); + + return true; +} + +bool BaseCompiler::emitStructNewDefault() { + uint32_t typeIndex; + if (!iter_.readStructNewDefault(&typeIndex)) { + return false; + } + if (deadCode_) { + return true; + } + + // Allocate a default initialized struct. This requires the type definition + // for the struct to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + return emitInstanceCall(SASigStructNew); +} + +bool BaseCompiler::emitStructGet(FieldWideningOp wideningOp) { + uint32_t typeIndex; + uint32_t fieldIndex; + Nothing nothing; + if (!iter_.readStructGet(&typeIndex, &fieldIndex, wideningOp, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const StructType& structType = (*moduleEnv_.types)[typeIndex].structType(); + + // Decide whether we're accessing inline or outline, and at what offset + FieldType fieldType = structType.fields_[fieldIndex].type; + uint32_t fieldOffset = structType.fields_[fieldIndex].offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + RegRef object = popRef(); + if (areaIsOutline) { + RegPtr outlineBase = needPtr(); + SignalNullCheck::emitTrapSite(this); + masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()), + outlineBase); + // Load the value + emitGcGet(fieldType, wideningOp, + Address(outlineBase, areaOffset)); + freePtr(outlineBase); + } else { + // Load the value + emitGcGet( + fieldType, wideningOp, + Address(object, WasmStructObject::offsetOfInlineData() + areaOffset)); + } + freeRef(object); + + return true; +} + +bool BaseCompiler::emitStructSet() { + uint32_t typeIndex; + uint32_t fieldIndex; + Nothing nothing; + if (!iter_.readStructSet(&typeIndex, &fieldIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const StructType& structType = (*moduleEnv_.types)[typeIndex].structType(); + const StructField& structField = structType.fields_[fieldIndex]; + + // Decide whether we're accessing inline or outline, and at what offset + FieldType fieldType = structType.fields_[fieldIndex].type; + uint32_t fieldOffset = structType.fields_[fieldIndex].offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (structField.type.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + + RegPtr outlineBase = areaIsOutline ? needPtr() : RegPtr(); + AnyReg value = popAny(); + RegRef object = popRef(); + + // Free the barrier reg after we've allocated all registers + if (structField.type.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + // Make outlineBase point at the first byte of the relevant area + if (areaIsOutline) { + SignalNullCheck::emitTrapSite(this); + masm.loadPtr(Address(object, WasmStructObject::offsetOfOutlineData()), + outlineBase); + if (!emitGcStructSet(object, outlineBase, areaOffset, + fieldType, value, + PreBarrierKind::Normal)) { + return false; + } + } else { + // Consumes value. object is unchanged by this call. + if (!emitGcStructSet( + object, RegPtr(object), + WasmStructObject::offsetOfInlineData() + areaOffset, fieldType, + value, PreBarrierKind::Normal)) { + return false; + } + } + + if (areaIsOutline) { + freePtr(outlineBase); + } + freeRef(object); + + return true; +} + +bool BaseCompiler::emitArrayNew() { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArrayNew(&typeIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // Allocate an uninitialized array. This requires the type definition + // for the array to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + if (!emitInstanceCall(SASigArrayNewUninit)) { + return false; + } + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (arrayType.elementType_.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + + RegRef rp = popRef(); + AnyReg value = popAny(); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements(rp); + + // Free the barrier reg after we've allocated all registers + if (arrayType.elementType_.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + // Perform an initialization loop using `numElements` as the loop variable, + // counting down to zero. + Label done; + Label loop; + // Skip initialization if numElements = 0 + masm.branch32(Assembler::Equal, numElements, Imm32(0), &done); + masm.bind(&loop); + + // Move to the next element + masm.sub32(Imm32(1), numElements); + + // Assign value to array[numElements]. All registers are preserved + if (!emitGcArraySet(rp, rdata, numElements, arrayType, value, + PreBarrierKind::None)) { + return false; + } + + // Loop back if there are still elements to initialize + masm.branch32(Assembler::GreaterThan, numElements, Imm32(0), &loop); + masm.bind(&done); + + freeI32(numElements); + freeAny(value); + freePtr(rdata); + pushRef(rp); + + return true; +} + +bool BaseCompiler::emitArrayNewFixed() { + uint32_t typeIndex, numElements; + BaseNothingVector nothings{}; + if (!iter_.readArrayNewFixed(&typeIndex, &numElements, ¬hings)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // At this point, the top section of the value stack contains the values to + // be used to initialise the array, with index 0 as the topmost value. Push + // the required number of elements and the required type on, since the call + // to SASigArrayNew will use them. + pushI32(numElements); + pushPtr(loadTypeDefInstanceData(typeIndex)); + if (!emitInstanceCall(SASigArrayNew)) { + return false; + } + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + bool avoidPreBarrierReg = arrayType.elementType_.isRefRepr(); + if (avoidPreBarrierReg) { + needPtr(RegPtr(PreBarrierReg)); + } + + // Get hold of the pointer to the array, as created by SASigArrayNew. + RegRef rp = popRef(); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Free the barrier reg if we previously reserved it. + if (avoidPreBarrierReg) { + freePtr(RegPtr(PreBarrierReg)); + } + + // These together ensure that the max value of `index` in the loop below + // remains comfortably below the 2^31 boundary. See comments on equivalent + // assertions in EmitArrayNewFixed in WasmIonCompile.cpp for explanation. + static_assert(16 /* sizeof v128 */ * MaxFunctionBytes <= + MaxArrayPayloadBytes); + MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes); + + // Generate straight-line initialization code. We could do better here if + // there was a version of ::emitGcArraySet that took `index` as a `uint32_t` + // rather than a general value-in-a-reg. + for (uint32_t forwardIndex = 0; forwardIndex < numElements; forwardIndex++) { + uint32_t reverseIndex = numElements - forwardIndex - 1; + if (avoidPreBarrierReg) { + needPtr(RegPtr(PreBarrierReg)); + } + AnyReg value = popAny(); + pushI32(reverseIndex); + RegI32 index = popI32(); + if (avoidPreBarrierReg) { + freePtr(RegPtr(PreBarrierReg)); + } + if (!emitGcArraySet(rp, rdata, index, arrayType, value, + PreBarrierKind::None)) { + return false; + } + freeI32(index); + freeAny(value); + } + + freePtr(rdata); + + pushRef(rp); + return true; +} + +bool BaseCompiler::emitArrayNewDefault() { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArrayNewDefault(&typeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + // Allocate a default initialized array. This requires the type definition + // for the array to be pushed on the stack. This will trap on OOM. + pushPtr(loadTypeDefInstanceData(typeIndex)); + return emitInstanceCall(SASigArrayNew); +} + +bool BaseCompiler::emitArrayNewData() { + uint32_t typeIndex, segIndex; + Nothing nothing; + if (!iter_.readArrayNewData(&typeIndex, &segIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushPtr(loadTypeDefInstanceData(typeIndex)); + pushI32(int32_t(segIndex)); + + // The call removes 4 items from the stack: the segment byte offset and + // number of elements (operands to array.new_data), and the type index and + // seg index as pushed above. + return emitInstanceCall(SASigArrayNewData); +} + +bool BaseCompiler::emitArrayNewElem() { + uint32_t typeIndex, segIndex; + Nothing nothing; + if (!iter_.readArrayNewElem(&typeIndex, &segIndex, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + pushPtr(loadTypeDefInstanceData(typeIndex)); + pushI32(int32_t(segIndex)); + + // The call removes 4 items from the stack: the segment element offset and + // number of elements (operands to array.new_elem), and the type index and + // seg index as pushed above. + return emitInstanceCall(SASigArrayNewElem); +} + +bool BaseCompiler::emitArrayGet(FieldWideningOp wideningOp) { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArrayGet(&typeIndex, wideningOp, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + RegI32 index = popI32(); + RegRef rp = popRef(); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements(rp); + + // Bounds check the index + emitGcArrayBoundsCheck(index, numElements); + freeI32(numElements); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Load the value + uint32_t shift = arrayType.elementType_.indexingShift(); + if (IsShiftInScaleRange(shift)) { + emitGcGet( + arrayType.elementType_, wideningOp, + BaseIndex(rdata, index, ShiftToScale(shift), 0)); + } else { + masm.lshiftPtr(Imm32(shift), index); + emitGcGet(arrayType.elementType_, wideningOp, + BaseIndex(rdata, index, TimesOne, 0)); + } + + freePtr(rdata); + freeRef(rp); + freeI32(index); + + return true; +} + +bool BaseCompiler::emitArraySet() { + uint32_t typeIndex; + Nothing nothing; + if (!iter_.readArraySet(&typeIndex, ¬hing, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (arrayType.elementType_.isRefRepr()) { + needPtr(RegPtr(PreBarrierReg)); + } + + AnyReg value = popAny(); + RegI32 index = popI32(); + RegRef rp = popRef(); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements(rp); + + // Bounds check the index + emitGcArrayBoundsCheck(index, numElements); + freeI32(numElements); + + // Acquire the data pointer from the object + RegPtr rdata = emitGcArrayGetData(rp); + + // Free the barrier reg after we've allocated all registers + if (arrayType.elementType_.isRefRepr()) { + freePtr(RegPtr(PreBarrierReg)); + } + + // All registers are preserved. This isn't strictly necessary, as we'll just + // be freeing them all after this is done. But this is needed for repeated + // assignments used in array.new/new_default. + if (!emitGcArraySet(rp, rdata, index, arrayType, value, + PreBarrierKind::Normal)) { + return false; + } + + freePtr(rdata); + freeRef(rp); + freeI32(index); + freeAny(value); + + return true; +} + +bool BaseCompiler::emitArrayLen(bool decodeIgnoredTypeIndex) { + Nothing nothing; + if (!iter_.readArrayLen(decodeIgnoredTypeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegRef rp = popRef(); + + // Acquire the number of elements + RegI32 numElements = emitGcArrayGetNumElements(rp); + pushI32(numElements); + + freeRef(rp); + + return true; +} + +bool BaseCompiler::emitArrayCopy() { + int32_t elemSize; + bool elemsAreRefTyped; + Nothing nothing; + if (!iter_.readArrayCopy(&elemSize, &elemsAreRefTyped, ¬hing, ¬hing, + ¬hing, ¬hing, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + // readArrayCopy writes the element size in bytes to `elemSize`. Code + // generated here needs to hand it onwards to the instance fn, but also + // indicate whether the element type is reftyped, which it does by negating + // the size. + // + // The value stack currently looks like this: + // + // (top) [numElements, srcIndex, srcArray, dstIndex, dstArray, ..] + // + // So push `(possibly negated) elemSize` on it, giving + // + // (top) [elemSize, numElements, srcIndex, srcArray, dstIndex, dstArray, ..] + // + // and generate a call to the helper. + + MOZ_ASSERT_IF(elemsAreRefTyped, elemSize == sizeof(void*)); + MOZ_ASSERT_IF(!elemsAreRefTyped, elemSize == 1 || elemSize == 2 || + elemSize == 4 || elemSize == 8 || + elemSize == 16); + bool avoidPreBarrierReg = elemsAreRefTyped; + + // Reserve this register early if we will need it so that it is not taken by + // any register used in this function. + if (avoidPreBarrierReg) { + needPtr(RegPtr(PreBarrierReg)); + } + + // The helper needs to know the element size. + pushI32(elemsAreRefTyped ? -elemSize : elemSize); + + if (avoidPreBarrierReg) { + freePtr(RegPtr(PreBarrierReg)); + } + + return emitInstanceCall(SASigArrayCopy); +} + +void BaseCompiler::emitRefTestCommon(RefType sourceType, RefType destType) { + Label success; + Label join; + RegRef object = popRef(); + RegI32 result = needI32(); + + branchGcRefType(object, sourceType, destType, &success, /*onSuccess=*/true); + masm.xor32(result, result); + masm.jump(&join); + masm.bind(&success); + masm.move32(Imm32(1), result); + masm.bind(&join); + + pushI32(result); + freeRef(object); +} + +void BaseCompiler::emitRefCastCommon(RefType sourceType, RefType destType) { + RegRef object = popRef(); + + Label success; + branchGcRefType(object, sourceType, destType, &success, /*onSuccess=*/true); + masm.wasmTrap(Trap::BadCast, bytecodeOffset()); + masm.bind(&success); + pushRef(object); +} + +bool BaseCompiler::emitRefTestV5() { + Nothing nothing; + RefType sourceType; + uint32_t typeIndex; + if (!iter_.readRefTestV5(&sourceType, &typeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TypeDef& typeDef = moduleEnv_.types->type(typeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/false); + emitRefTestCommon(sourceType, destType); + + return true; +} + +void BaseCompiler::branchGcRefType(RegRef object, RefType sourceType, + RefType destType, Label* label, + bool onSuccess) { + RegPtr superSuperTypeVector; + if (MacroAssembler::needSuperSuperTypeVectorForBranchWasmGcRefType( + destType)) { + uint32_t typeIndex = moduleEnv_.types->indexOf(*destType.typeDef()); + superSuperTypeVector = loadSuperTypeVector(typeIndex); + } + RegI32 scratch1 = MacroAssembler::needScratch1ForBranchWasmGcRefType(destType) + ? needI32() + : RegI32::Invalid(); + RegI32 scratch2 = MacroAssembler::needScratch2ForBranchWasmGcRefType(destType) + ? needI32() + : RegI32::Invalid(); + + masm.branchWasmGcObjectIsRefType(object, sourceType, destType, label, + onSuccess, superSuperTypeVector, scratch1, + scratch2); + + if (scratch2.isValid()) { + freeI32(scratch2); + } + if (scratch1.isValid()) { + freeI32(scratch1); + } + if (superSuperTypeVector.isValid()) { + freePtr(superSuperTypeVector); + } +} + +bool BaseCompiler::emitRefCastV5() { + Nothing nothing; + RefType sourceType; + uint32_t typeIndex; + if (!iter_.readRefCastV5(&sourceType, &typeIndex, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TypeDef& typeDef = moduleEnv_.types->type(typeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/true); + emitRefCastCommon(sourceType, destType); + + return true; +} + +bool BaseCompiler::emitRefTest(bool nullable) { + Nothing nothing; + RefType sourceType; + RefType destType; + if (!iter_.readRefTest(nullable, &sourceType, &destType, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + emitRefTestCommon(sourceType, destType); + + return true; +} + +bool BaseCompiler::emitRefCast(bool nullable) { + Nothing nothing; + RefType sourceType; + RefType destType; + if (!iter_.readRefCast(nullable, &sourceType, &destType, ¬hing)) { + return false; + } + + if (deadCode_) { + return true; + } + + emitRefCastCommon(sourceType, destType); + + return true; +} + +bool BaseCompiler::emitBrOnCastCommon(bool onSuccess, + uint32_t labelRelativeDepth, + const ResultType& labelType, + RefType sourceType, RefType destType) { + Control& target = controlItem(labelRelativeDepth); + target.bceSafeOnExit &= bceSafe_; + + // 3. br_if $l : [T*, ref] -> [T*, ref] + BranchState b(&target.label, target.stackHeight, InvertBranch(false), + labelType); + + // Don't allocate the result register used in the branch + if (b.hasBlockResults()) { + needIntegerResultRegisters(b.resultType); + } + + // Create a copy of the ref for passing to the br_on_cast label, + // the original ref is used for casting in the condition. + RegRef object = popRef(); + RegRef objectCondition = needRef(); + moveRef(object, objectCondition); + pushRef(object); + + if (b.hasBlockResults()) { + freeIntegerResultRegisters(b.resultType); + } + + if (!jumpConditionalWithResults(&b, objectCondition, sourceType, destType, + onSuccess)) { + return false; + } + freeRef(objectCondition); + + return true; +} + +bool BaseCompiler::emitBrOnCast() { + MOZ_ASSERT(!hasLatentOp()); + + bool onSuccess; + uint32_t labelRelativeDepth; + RefType sourceType; + RefType destType; + ResultType labelType; + BaseNothingVector unused_values{}; + if (!iter_.readBrOnCast(&onSuccess, &labelRelativeDepth, &sourceType, + &destType, &labelType, &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType, + sourceType, destType); +} + +bool BaseCompiler::emitBrOnCastV5(bool onSuccess) { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t labelRelativeDepth; + RefType sourceType; + uint32_t castTypeIndex; + ResultType labelType; + BaseNothingVector unused_values{}; + if (onSuccess + ? !iter_.readBrOnCastV5(&labelRelativeDepth, &sourceType, + &castTypeIndex, &labelType, &unused_values) + : !iter_.readBrOnCastFailV5(&labelRelativeDepth, &sourceType, + &castTypeIndex, &labelType, + &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + const TypeDef& typeDef = moduleEnv_.types->type(castTypeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, false); + return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType, + sourceType, destType); +} + +bool BaseCompiler::emitBrOnCastHeapV5(bool onSuccess, bool nullable) { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t labelRelativeDepth; + RefType sourceType; + RefType destType; + ResultType labelType; + BaseNothingVector unused_values{}; + if (onSuccess ? !iter_.readBrOnCastHeapV5(nullable, &labelRelativeDepth, + &sourceType, &destType, &labelType, + &unused_values) + : !iter_.readBrOnCastFailHeapV5(nullable, &labelRelativeDepth, + &sourceType, &destType, + &labelType, &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + return emitBrOnCastCommon(onSuccess, labelRelativeDepth, labelType, + sourceType, destType); +} + +bool BaseCompiler::emitRefAsStructV5() { + Nothing nothing; + return iter_.readConversion(ValType(RefType::any()), + ValType(RefType::struct_().asNonNullable()), + ¬hing); +} + +bool BaseCompiler::emitBrOnNonStructV5() { + MOZ_ASSERT(!hasLatentOp()); + + uint32_t labelRelativeDepth; + ResultType labelType; + BaseNothingVector unused_values{}; + if (!iter_.readBrOnNonStructV5(&labelRelativeDepth, &labelType, + &unused_values)) { + return false; + } + + if (deadCode_) { + return true; + } + + Control& target = controlItem(labelRelativeDepth); + target.bceSafeOnExit &= bceSafe_; + + BranchState b(&target.label, target.stackHeight, InvertBranch(false), + labelType); + if (b.hasBlockResults()) { + needResultRegisters(b.resultType); + } + RegI32 condition = needI32(); + masm.move32(Imm32(1), condition); + if (b.hasBlockResults()) { + freeResultRegisters(b.resultType); + } + if (!jumpConditionalWithResults(&b, Assembler::Equal, condition, Imm32(0))) { + return false; + } + freeI32(condition); + return true; +} + +bool BaseCompiler::emitExternInternalize() { + // extern.internalize is a no-op because anyref and extern share the same + // representation + Nothing nothing; + return iter_.readRefConversion(RefType::extern_(), RefType::any(), ¬hing); +} + +bool BaseCompiler::emitExternExternalize() { + // extern.externalize is a no-op because anyref and extern share the same + // representation + Nothing nothing; + return iter_.readRefConversion(RefType::any(), RefType::extern_(), ¬hing); +} + +#endif // ENABLE_WASM_GC + +////////////////////////////////////////////////////////////////////////////// +// +// SIMD and Relaxed SIMD. + +#ifdef ENABLE_WASM_SIMD + +// Emitter trampolines used by abstracted SIMD operations. Naming here follows +// the SIMD spec pretty closely. + +static void AndV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.bitwiseAndSimd128(rs, rsd); +} + +static void OrV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.bitwiseOrSimd128(rs, rsd); +} + +static void XorV128(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.bitwiseXorSimd128(rs, rsd); +} + +static void AddI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt8x16(rsd, rs, rsd); +} + +static void AddI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt16x8(rsd, rs, rsd); +} + +static void AddI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt32x4(rsd, rs, rsd); +} + +static void AddF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addFloat32x4(rsd, rs, rsd); +} + +static void AddI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addInt64x2(rsd, rs, rsd); +} + +static void AddF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addFloat64x2(rsd, rs, rsd); +} + +static void AddSatI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addSatInt8x16(rsd, rs, rsd); +} + +static void AddSatUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAddSatInt8x16(rsd, rs, rsd); +} + +static void AddSatI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.addSatInt16x8(rsd, rs, rsd); +} + +static void AddSatUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAddSatInt16x8(rsd, rs, rsd); +} + +static void SubI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt8x16(rsd, rs, rsd); +} + +static void SubI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt16x8(rsd, rs, rsd); +} + +static void SubI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt32x4(rsd, rs, rsd); +} + +static void SubF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subFloat32x4(rsd, rs, rsd); +} + +static void SubI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subInt64x2(rsd, rs, rsd); +} + +static void SubF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subFloat64x2(rsd, rs, rsd); +} + +static void SubSatI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subSatInt8x16(rsd, rs, rsd); +} + +static void SubSatUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedSubSatInt8x16(rsd, rs, rsd); +} + +static void SubSatI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.subSatInt16x8(rsd, rs, rsd); +} + +static void SubSatUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedSubSatInt16x8(rsd, rs, rsd); +} + +static void MulI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulInt16x8(rsd, rs, rsd); +} + +static void MulI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulInt32x4(rsd, rs, rsd); +} + +static void MulF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulFloat32x4(rsd, rs, rsd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void MulI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp) { + masm.mulInt64x2(rsd, rs, rsd, temp); +} +# elif defined(JS_CODEGEN_ARM64) +static void MulI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.mulInt64x2(rsd, rs, rsd, temp1, temp2); +} +# endif + +static void MulF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.mulFloat64x2(rsd, rs, rsd); +} + +static void DivF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.divFloat32x4(rsd, rs, rsd); +} + +static void DivF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.divFloat64x2(rsd, rs, rsd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void MinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.minFloat32x4(rsd, rs, rsd, temp1, temp2); +} + +static void MinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.minFloat64x2(rsd, rs, rsd, temp1, temp2); +} + +static void MaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.maxFloat32x4(rsd, rs, rsd, temp1, temp2); +} + +static void MaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd, + RegV128 temp1, RegV128 temp2) { + masm.maxFloat64x2(rsd, rs, rsd, temp1, temp2); +} + +static void PMinF32x4(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMinFloat32x4(rsd, rs); +} + +static void PMinF64x2(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMinFloat64x2(rsd, rs); +} + +static void PMaxF32x4(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMaxFloat32x4(rsd, rs); +} + +static void PMaxF64x2(MacroAssembler& masm, RegV128 rsd, RegV128 rs, + RhsDestOp) { + masm.pseudoMaxFloat64x2(rsd, rs); +} +# elif defined(JS_CODEGEN_ARM64) +static void MinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat32x4(rs, rsd); +} + +static void MinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat64x2(rs, rsd); +} + +static void MaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat32x4(rs, rsd); +} + +static void MaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat64x2(rs, rsd); +} + +static void PMinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMinFloat32x4(rs, rsd); +} + +static void PMinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMinFloat64x2(rs, rsd); +} + +static void PMaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMaxFloat32x4(rs, rsd); +} + +static void PMaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.pseudoMaxFloat64x2(rs, rsd); +} +# endif + +static void DotI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.widenDotInt16x8(rsd, rs, rsd); +} + +static void ExtMulLowI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulLowInt8x16(rsd, rs, rsd); +} + +static void ExtMulHighI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulHighInt8x16(rsd, rs, rsd); +} + +static void ExtMulLowUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulLowInt8x16(rsd, rs, rsd); +} + +static void ExtMulHighUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulHighInt8x16(rsd, rs, rsd); +} + +static void ExtMulLowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulLowInt16x8(rsd, rs, rsd); +} + +static void ExtMulHighI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulHighInt16x8(rsd, rs, rsd); +} + +static void ExtMulLowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulLowInt16x8(rsd, rs, rsd); +} + +static void ExtMulHighUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulHighInt16x8(rsd, rs, rsd); +} + +static void ExtMulLowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulLowInt32x4(rsd, rs, rsd); +} + +static void ExtMulHighI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extMulHighInt32x4(rsd, rs, rsd); +} + +static void ExtMulLowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulLowInt32x4(rsd, rs, rsd); +} + +static void ExtMulHighUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedExtMulHighInt32x4(rsd, rs, rsd); +} + +static void Q15MulrSatS(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.q15MulrSatInt16x8(rsd, rs, rsd); +} + +static void CmpI8x16(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt8x16(cond, rs, rsd); +} + +static void CmpI16x8(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt16x8(cond, rs, rsd); +} + +static void CmpI32x4(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt32x4(cond, rs, rsd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void CmpI64x2ForEquality(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareForEqualityInt64x2(cond, rsd, rs, rsd); +} + +static void CmpI64x2ForOrdering(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd, RegV128 temp1, + RegV128 temp2) { + masm.compareForOrderingInt64x2(cond, rsd, rs, rsd, temp1, temp2); +} +# else +static void CmpI64x2ForEquality(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt64x2(cond, rs, rsd); +} + +static void CmpI64x2ForOrdering(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt64x2(cond, rs, rsd); +} +# endif // JS_CODEGEN_X86 || JS_CODEGEN_X64 + +static void CmpUI8x16(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt8x16(cond, rs, rsd); +} + +static void CmpUI16x8(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt16x8(cond, rs, rsd); +} + +static void CmpUI32x4(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareInt32x4(cond, rs, rsd); +} + +static void CmpF32x4(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareFloat32x4(cond, rs, rsd); +} + +static void CmpF64x2(MacroAssembler& masm, Assembler::Condition cond, + RegV128 rs, RegV128 rsd) { + masm.compareFloat64x2(cond, rs, rsd); +} + +static void NegI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt8x16(rs, rd); +} + +static void NegI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt16x8(rs, rd); +} + +static void NegI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt32x4(rs, rd); +} + +static void NegI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negInt64x2(rs, rd); +} + +static void NegF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negFloat32x4(rs, rd); +} + +static void NegF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.negFloat64x2(rs, rd); +} + +static void AbsF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absFloat32x4(rs, rd); +} + +static void AbsF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absFloat64x2(rs, rd); +} + +static void SqrtF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.sqrtFloat32x4(rs, rd); +} + +static void SqrtF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.sqrtFloat64x2(rs, rd); +} + +static void CeilF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.ceilFloat32x4(rs, rd); +} + +static void FloorF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.floorFloat32x4(rs, rd); +} + +static void TruncF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.truncFloat32x4(rs, rd); +} + +static void NearestF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.nearestFloat32x4(rs, rd); +} + +static void CeilF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.ceilFloat64x2(rs, rd); +} + +static void FloorF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.floorFloat64x2(rs, rd); +} + +static void TruncF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.truncFloat64x2(rs, rd); +} + +static void NearestF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.nearestFloat64x2(rs, rd); +} + +static void NotV128(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.bitwiseNotSimd128(rs, rd); +} + +static void ExtAddPairwiseI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extAddPairwiseInt8x16(rs, rsd); +} + +static void ExtAddPairwiseUI8x16(MacroAssembler& masm, RegV128 rs, + RegV128 rsd) { + masm.unsignedExtAddPairwiseInt8x16(rs, rsd); +} + +static void ExtAddPairwiseI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.extAddPairwiseInt16x8(rs, rsd); +} + +static void ExtAddPairwiseUI16x8(MacroAssembler& masm, RegV128 rs, + RegV128 rsd) { + masm.unsignedExtAddPairwiseInt16x8(rs, rsd); +} + +static void ShiftOpMask(MacroAssembler& masm, SimdOp op, RegI32 in, + RegI32 out) { + int32_t maskBits; + + masm.mov(in, out); + if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) { + masm.and32(Imm32(maskBits), out); + } +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void ShiftLeftI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp1, RegV128 temp2) { + ShiftOpMask(masm, SimdOp::I8x16Shl, rs, temp1); + masm.leftShiftInt8x16(temp1, rsd, temp2); +} + +static void ShiftLeftI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8Shl, rs, temp); + masm.leftShiftInt16x8(temp, rsd); +} + +static void ShiftLeftI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4Shl, rs, temp); + masm.leftShiftInt32x4(temp, rsd); +} + +static void ShiftLeftI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2Shl, rs, temp); + masm.leftShiftInt64x2(temp, rsd); +} + +static void ShiftRightI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp1, RegV128 temp2) { + ShiftOpMask(masm, SimdOp::I8x16ShrS, rs, temp1); + masm.rightShiftInt8x16(temp1, rsd, temp2); +} + +static void ShiftRightUI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp1, RegV128 temp2) { + ShiftOpMask(masm, SimdOp::I8x16ShrU, rs, temp1); + masm.unsignedRightShiftInt8x16(temp1, rsd, temp2); +} + +static void ShiftRightI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrS, rs, temp); + masm.rightShiftInt16x8(temp, rsd); +} + +static void ShiftRightUI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrU, rs, temp); + masm.unsignedRightShiftInt16x8(temp, rsd); +} + +static void ShiftRightI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrS, rs, temp); + masm.rightShiftInt32x4(temp, rsd); +} + +static void ShiftRightUI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrU, rs, temp); + masm.unsignedRightShiftInt32x4(temp, rsd); +} + +static void ShiftRightUI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2ShrU, rs, temp); + masm.unsignedRightShiftInt64x2(temp, rsd); +} +# elif defined(JS_CODEGEN_ARM64) +static void ShiftLeftI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I8x16Shl, rs, temp); + masm.leftShiftInt8x16(rsd, temp, rsd); +} + +static void ShiftLeftI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8Shl, rs, temp); + masm.leftShiftInt16x8(rsd, temp, rsd); +} + +static void ShiftLeftI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4Shl, rs, temp); + masm.leftShiftInt32x4(rsd, temp, rsd); +} + +static void ShiftLeftI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2Shl, rs, temp); + masm.leftShiftInt64x2(rsd, temp, rsd); +} + +static void ShiftRightI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I8x16ShrS, rs, temp); + masm.rightShiftInt8x16(rsd, temp, rsd); +} + +static void ShiftRightUI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I8x16ShrU, rs, temp); + masm.unsignedRightShiftInt8x16(rsd, temp, rsd); +} + +static void ShiftRightI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrS, rs, temp); + masm.rightShiftInt16x8(rsd, temp, rsd); +} + +static void ShiftRightUI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I16x8ShrU, rs, temp); + masm.unsignedRightShiftInt16x8(rsd, temp, rsd); +} + +static void ShiftRightI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrS, rs, temp); + masm.rightShiftInt32x4(rsd, temp, rsd); +} + +static void ShiftRightUI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I32x4ShrU, rs, temp); + masm.unsignedRightShiftInt32x4(rsd, temp, rsd); +} + +static void ShiftRightI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2ShrS, rs, temp); + masm.rightShiftInt64x2(rsd, temp, rsd); +} + +static void ShiftRightUI64x2(MacroAssembler& masm, RegI32 rs, RegV128 rsd, + RegI32 temp) { + ShiftOpMask(masm, SimdOp::I64x2ShrU, rs, temp); + masm.unsignedRightShiftInt64x2(rsd, temp, rsd); +} +# endif + +static void AverageUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAverageInt8x16(rsd, rs, rsd); +} + +static void AverageUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedAverageInt16x8(rsd, rs, rsd); +} + +static void MinI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minInt8x16(rsd, rs, rsd); +} + +static void MinUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMinInt8x16(rsd, rs, rsd); +} + +static void MaxI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxInt8x16(rsd, rs, rsd); +} + +static void MaxUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMaxInt8x16(rsd, rs, rsd); +} + +static void MinI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minInt16x8(rsd, rs, rsd); +} + +static void MinUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMinInt16x8(rsd, rs, rsd); +} + +static void MaxI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxInt16x8(rsd, rs, rsd); +} + +static void MaxUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMaxInt16x8(rsd, rs, rsd); +} + +static void MinI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minInt32x4(rsd, rs, rsd); +} + +static void MinUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMinInt32x4(rsd, rs, rsd); +} + +static void MaxI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxInt32x4(rsd, rs, rsd); +} + +static void MaxUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedMaxInt32x4(rsd, rs, rsd); +} + +static void NarrowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.narrowInt16x8(rsd, rs, rsd); +} + +static void NarrowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedNarrowInt16x8(rsd, rs, rsd); +} + +static void NarrowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.narrowInt32x4(rsd, rs, rsd); +} + +static void NarrowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.unsignedNarrowInt32x4(rsd, rs, rsd); +} + +static void WidenLowI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenLowInt8x16(rs, rd); +} + +static void WidenHighI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenHighInt8x16(rs, rd); +} + +static void WidenLowUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenLowInt8x16(rs, rd); +} + +static void WidenHighUI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenHighInt8x16(rs, rd); +} + +static void WidenLowI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenLowInt16x8(rs, rd); +} + +static void WidenHighI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenHighInt16x8(rs, rd); +} + +static void WidenLowUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenLowInt16x8(rs, rd); +} + +static void WidenHighUI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenHighInt16x8(rs, rd); +} + +static void WidenLowI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenLowInt32x4(rs, rd); +} + +static void WidenHighI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.widenHighInt32x4(rs, rd); +} + +static void WidenLowUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenLowInt32x4(rs, rd); +} + +static void WidenHighUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedWidenHighInt32x4(rs, rd); +} + +# if defined(JS_CODEGEN_ARM64) +static void PopcntI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.popcntInt8x16(rs, rd); +} +# else +static void PopcntI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.popcntInt8x16(rs, rd, temp); +} +# endif // JS_CODEGEN_ARM64 + +static void AbsI8x16(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt8x16(rs, rd); +} + +static void AbsI16x8(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt16x8(rs, rd); +} + +static void AbsI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt32x4(rs, rd); +} + +static void AbsI64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.absInt64x2(rs, rd); +} + +static void ExtractLaneI8x16(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.extractLaneInt8x16(laneIndex, rs, rd); +} + +static void ExtractLaneUI8x16(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.unsignedExtractLaneInt8x16(laneIndex, rs, rd); +} + +static void ExtractLaneI16x8(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.extractLaneInt16x8(laneIndex, rs, rd); +} + +static void ExtractLaneUI16x8(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.unsignedExtractLaneInt16x8(laneIndex, rs, rd); +} + +static void ExtractLaneI32x4(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI32 rd) { + masm.extractLaneInt32x4(laneIndex, rs, rd); +} + +static void ExtractLaneI64x2(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegI64 rd) { + masm.extractLaneInt64x2(laneIndex, rs, rd); +} + +static void ExtractLaneF32x4(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegF32 rd) { + masm.extractLaneFloat32x4(laneIndex, rs, rd); +} + +static void ExtractLaneF64x2(MacroAssembler& masm, uint32_t laneIndex, + RegV128 rs, RegF64 rd) { + masm.extractLaneFloat64x2(laneIndex, rs, rd); +} + +static void ReplaceLaneI8x16(MacroAssembler& masm, uint32_t laneIndex, + RegI32 rs, RegV128 rsd) { + masm.replaceLaneInt8x16(laneIndex, rs, rsd); +} + +static void ReplaceLaneI16x8(MacroAssembler& masm, uint32_t laneIndex, + RegI32 rs, RegV128 rsd) { + masm.replaceLaneInt16x8(laneIndex, rs, rsd); +} + +static void ReplaceLaneI32x4(MacroAssembler& masm, uint32_t laneIndex, + RegI32 rs, RegV128 rsd) { + masm.replaceLaneInt32x4(laneIndex, rs, rsd); +} + +static void ReplaceLaneI64x2(MacroAssembler& masm, uint32_t laneIndex, + RegI64 rs, RegV128 rsd) { + masm.replaceLaneInt64x2(laneIndex, rs, rsd); +} + +static void ReplaceLaneF32x4(MacroAssembler& masm, uint32_t laneIndex, + RegF32 rs, RegV128 rsd) { + masm.replaceLaneFloat32x4(laneIndex, rs, rsd); +} + +static void ReplaceLaneF64x2(MacroAssembler& masm, uint32_t laneIndex, + RegF64 rs, RegV128 rsd) { + masm.replaceLaneFloat64x2(laneIndex, rs, rsd); +} + +static void SplatI8x16(MacroAssembler& masm, RegI32 rs, RegV128 rd) { + masm.splatX16(rs, rd); +} + +static void SplatI16x8(MacroAssembler& masm, RegI32 rs, RegV128 rd) { + masm.splatX8(rs, rd); +} + +static void SplatI32x4(MacroAssembler& masm, RegI32 rs, RegV128 rd) { + masm.splatX4(rs, rd); +} + +static void SplatI64x2(MacroAssembler& masm, RegI64 rs, RegV128 rd) { + masm.splatX2(rs, rd); +} + +static void SplatF32x4(MacroAssembler& masm, RegF32 rs, RegV128 rd) { + masm.splatX4(rs, rd); +} + +static void SplatF64x2(MacroAssembler& masm, RegF64 rs, RegV128 rd) { + masm.splatX2(rs, rd); +} + +// This is the same op independent of lanes: it tests for any nonzero bit. +static void AnyTrue(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.anyTrueSimd128(rs, rd); +} + +static void AllTrueI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt8x16(rs, rd); +} + +static void AllTrueI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt16x8(rs, rd); +} + +static void AllTrueI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt32x4(rs, rd); +} + +static void AllTrueI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.allTrueInt64x2(rs, rd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void BitmaskI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt8x16(rs, rd); +} + +static void BitmaskI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt16x8(rs, rd); +} + +static void BitmaskI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt32x4(rs, rd); +} + +static void BitmaskI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd) { + masm.bitmaskInt64x2(rs, rd); +} +# elif defined(JS_CODEGEN_ARM64) +static void BitmaskI8x16(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt8x16(rs, rd, temp); +} + +static void BitmaskI16x8(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt16x8(rs, rd, temp); +} + +static void BitmaskI32x4(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt32x4(rs, rd, temp); +} + +static void BitmaskI64x2(MacroAssembler& masm, RegV128 rs, RegI32 rd, + RegV128 temp) { + masm.bitmaskInt64x2(rs, rd, temp); +} +# endif + +static void Swizzle(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.swizzleInt8x16(rsd, rs, rsd); +} + +static void ConvertI32x4ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertInt32x4ToFloat32x4(rs, rd); +} + +static void ConvertUI32x4ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedConvertInt32x4ToFloat32x4(rs, rd); +} + +static void ConvertF32x4ToI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.truncSatFloat32x4ToInt32x4(rs, rd); +} + +# if defined(JS_CODEGEN_ARM64) +static void ConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedTruncSatFloat32x4ToInt32x4(rs, rd); +} +# else +static void ConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.unsignedTruncSatFloat32x4ToInt32x4(rs, rd, temp); +} +# endif // JS_CODEGEN_ARM64 + +static void ConvertI32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertInt32x4ToFloat64x2(rs, rd); +} + +static void ConvertUI32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.unsignedConvertInt32x4ToFloat64x2(rs, rd); +} + +static void ConvertF64x2ToI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.truncSatFloat64x2ToInt32x4(rs, rd, temp); +} + +static void ConvertF64x2ToUI32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd, + RegV128 temp) { + masm.unsignedTruncSatFloat64x2ToInt32x4(rs, rd, temp); +} + +static void DemoteF64x2ToF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertFloat64x2ToFloat32x4(rs, rd); +} + +static void PromoteF32x4ToF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rd) { + masm.convertFloat32x4ToFloat64x2(rs, rd); +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +static void BitselectV128(MacroAssembler& masm, RegV128 rhs, RegV128 control, + RegV128 lhsDest, RegV128 temp) { + // Ideally, we would have temp=control, and we can probably get away with + // just doing that, but don't worry about it yet. + masm.bitwiseSelectSimd128(control, lhsDest, rhs, lhsDest, temp); +} +# elif defined(JS_CODEGEN_ARM64) +static void BitselectV128(MacroAssembler& masm, RegV128 rhs, RegV128 control, + RegV128 lhsDest, RegV128 temp) { + // The masm interface is not great for the baseline compiler here, but it's + // optimal for Ion, so just work around it. + masm.moveSimd128(control, temp); + masm.bitwiseSelectSimd128(lhsDest, rhs, temp); + masm.moveSimd128(temp, lhsDest); +} +# endif + +# ifdef ENABLE_WASM_RELAXED_SIMD +static void RelaxedFmaF32x4(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fmaFloat32x4(rs1, rs2, rsd); +} + +static void RelaxedFnmaF32x4(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fnmaFloat32x4(rs1, rs2, rsd); +} + +static void RelaxedFmaF64x2(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fmaFloat64x2(rs1, rs2, rsd); +} + +static void RelaxedFnmaF64x2(MacroAssembler& masm, RegV128 rs1, RegV128 rs2, + RegV128 rsd) { + masm.fnmaFloat64x2(rs1, rs2, rsd); +} + +static void RelaxedSwizzle(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.swizzleInt8x16Relaxed(rsd, rs, rsd); +} + +static void RelaxedMinF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat32x4Relaxed(rs, rsd); +} + +static void RelaxedMaxF32x4(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat32x4Relaxed(rs, rsd); +} + +static void RelaxedMinF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.minFloat64x2Relaxed(rs, rsd); +} + +static void RelaxedMaxF64x2(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.maxFloat64x2Relaxed(rs, rsd); +} + +static void RelaxedConvertF32x4ToI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.truncFloat32x4ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedConvertF32x4ToUI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.unsignedTruncFloat32x4ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedConvertF64x2ToI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.truncFloat64x2ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedConvertF64x2ToUI32x4(MacroAssembler& masm, RegV128 rs, + RegV128 rd) { + masm.unsignedTruncFloat64x2ToInt32x4Relaxed(rs, rd); +} + +static void RelaxedQ15MulrS(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.q15MulrInt16x8Relaxed(rsd, rs, rsd); +} + +static void DotI8x16I7x16S(MacroAssembler& masm, RegV128 rs, RegV128 rsd) { + masm.dotInt8x16Int7x16(rsd, rs, rsd); +} + +void BaseCompiler::emitDotI8x16I7x16AddS() { + RegV128 rsd = popV128(); + RegV128 rs0, rs1; + pop2xV128(&rs0, &rs1); +# if defined(JS_CODEGEN_ARM64) + RegV128 temp = needV128(); + masm.dotInt8x16Int7x16ThenAdd(rs0, rs1, rsd, temp); + freeV128(temp); +# else + masm.dotInt8x16Int7x16ThenAdd(rs0, rs1, rsd); +# endif + freeV128(rs1); + freeV128(rs0); + pushV128(rsd); +} +# endif // ENABLE_WASM_RELAXED_SIMD + +void BaseCompiler::emitVectorAndNot() { + // We want x & ~y but the available operation is ~x & y, so reverse the + // operands. + RegV128 r, rs; + pop2xV128(&r, &rs); + masm.bitwiseNotAndSimd128(r, rs); + freeV128(r); + pushV128(rs); +} + +bool BaseCompiler::emitLoadSplat(Scalar::Type viewType) { + LinearMemoryAddress addr; + if (!iter_.readLoadSplat(Scalar::byteSize(viewType), &addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadSplat(&access); + return true; +} + +bool BaseCompiler::emitLoadZero(Scalar::Type viewType) { + // LoadZero has the structure of LoadSplat, so reuse the reader. + LinearMemoryAddress addr; + if (!iter_.readLoadSplat(Scalar::byteSize(viewType), &addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadZero(&access); + return true; +} + +bool BaseCompiler::emitLoadExtend(Scalar::Type viewType) { + LinearMemoryAddress addr; + if (!iter_.readLoadExtend(&addr)) { + return false; + } + if (deadCode_) { + return true; + } + MemoryAccessDesc access(Scalar::Int64, addr.align, addr.offset, + bytecodeOffset()); + loadExtend(&access, viewType); + return true; +} + +bool BaseCompiler::emitLoadLane(uint32_t laneSize) { + Nothing nothing; + LinearMemoryAddress addr; + uint32_t laneIndex; + if (!iter_.readLoadLane(laneSize, &addr, &laneIndex, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + Scalar::Type viewType; + switch (laneSize) { + case 1: + viewType = Scalar::Uint8; + break; + case 2: + viewType = Scalar::Uint16; + break; + case 4: + viewType = Scalar::Int32; + break; + case 8: + viewType = Scalar::Int64; + break; + default: + MOZ_CRASH("unsupported laneSize"); + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + loadLane(&access, laneIndex); + return true; +} + +bool BaseCompiler::emitStoreLane(uint32_t laneSize) { + Nothing nothing; + LinearMemoryAddress addr; + uint32_t laneIndex; + if (!iter_.readStoreLane(laneSize, &addr, &laneIndex, ¬hing)) { + return false; + } + if (deadCode_) { + return true; + } + Scalar::Type viewType; + switch (laneSize) { + case 1: + viewType = Scalar::Uint8; + break; + case 2: + viewType = Scalar::Uint16; + break; + case 4: + viewType = Scalar::Int32; + break; + case 8: + viewType = Scalar::Int64; + break; + default: + MOZ_CRASH("unsupported laneSize"); + } + MemoryAccessDesc access(viewType, addr.align, addr.offset, bytecodeOffset()); + storeLane(&access, laneIndex); + return true; +} + +bool BaseCompiler::emitVectorShuffle() { + Nothing unused_a, unused_b; + V128 shuffleMask; + + if (!iter_.readVectorShuffle(&unused_a, &unused_b, &shuffleMask)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegV128 rd, rs; + pop2xV128(&rd, &rs); + + masm.shuffleInt8x16(shuffleMask.bytes, rs, rd); + + freeV128(rs); + pushV128(rd); + + return true; +} + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +bool BaseCompiler::emitVectorShiftRightI64x2() { + Nothing unused_a, unused_b; + + if (!iter_.readVectorShift(&unused_a, &unused_b)) { + return false; + } + + if (deadCode_) { + return true; + } + + RegI32 count = popI32RhsForShiftI64(); + RegV128 lhsDest = popV128(); + RegI64 tmp = needI64(); + masm.and32(Imm32(63), count); + masm.extractLaneInt64x2(0, lhsDest, tmp); + masm.rshift64Arithmetic(count, tmp); + masm.replaceLaneInt64x2(0, tmp, lhsDest); + masm.extractLaneInt64x2(1, lhsDest, tmp); + masm.rshift64Arithmetic(count, tmp); + masm.replaceLaneInt64x2(1, tmp, lhsDest); + freeI64(tmp); + freeI32(count); + pushV128(lhsDest); + + return true; +} +# endif +#endif // ENABLE_WASM_SIMD + +#ifdef ENABLE_WASM_RELAXED_SIMD +bool BaseCompiler::emitVectorLaneSelect() { + Nothing unused_a, unused_b, unused_c; + + if (!iter_.readTernary(ValType::V128, &unused_a, &unused_b, &unused_c)) { + return false; + } + + if (deadCode_) { + return true; + } + +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + RegV128 mask = popV128(RegV128(vmm0)); + RegV128 rhsDest = popV128(); + RegV128 lhs = popV128(); + masm.laneSelectSimd128(mask, lhs, rhsDest, rhsDest); + freeV128(lhs); + freeV128(mask); + pushV128(rhsDest); +# elif defined(JS_CODEGEN_ARM64) + RegV128 maskDest = popV128(); + RegV128 rhs = popV128(); + RegV128 lhs = popV128(); + masm.laneSelectSimd128(maskDest, lhs, rhs, maskDest); + freeV128(lhs); + freeV128(rhs); + pushV128(maskDest); +# endif + + return true; +} +#endif // ENABLE_WASM_RELAXED_SIMD + +////////////////////////////////////////////////////////////////////////////// +// +// "Intrinsics" - magically imported functions for internal use. + +bool BaseCompiler::emitIntrinsic() { + const Intrinsic* intrinsic; + + BaseNothingVector params; + if (!iter_.readIntrinsic(&intrinsic, ¶ms)) { + return false; + } + + if (deadCode_) { + return true; + } + + // The final parameter of an intrinsic is implicitly the heap base + pushHeapBase(); + + // Call the intrinsic + return emitInstanceCall(intrinsic->signature); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Function bodies - main opcode dispatch loop. + +bool BaseCompiler::emitBody() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::emitBody"); + + MOZ_ASSERT(stackMapGenerator_.framePushedAtEntryToBody.isSome()); + + if (!iter_.startFunction(func_.index, locals_)) { + return false; + } + + initControl(controlItem(), ResultType::Empty()); + + for (;;) { + Nothing unused_a, unused_b, unused_c; + (void)unused_a; + (void)unused_b; + (void)unused_c; + +#ifdef DEBUG + performRegisterLeakCheck(); + assertStackInvariants(); +#endif + +#define dispatchBinary0(doEmit, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (doEmit(), true)) + +#define dispatchBinary1(arg1, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(arg1), true)) + +#define dispatchBinary2(arg1, arg2, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(arg1, arg2), true)) + +#define dispatchBinary3(arg1, arg2, arg3, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(arg1, arg2, arg3), true)) + +#define dispatchUnary0(doEmit, type) \ + iter_.readUnary(type, &unused_a) && (deadCode_ || (doEmit(), true)) + +#define dispatchUnary1(arg1, type) \ + iter_.readUnary(type, &unused_a) && (deadCode_ || (emitUnop(arg1), true)) + +#define dispatchUnary2(arg1, arg2, type) \ + iter_.readUnary(type, &unused_a) && \ + (deadCode_ || (emitUnop(arg1, arg2), true)) + +#define dispatchTernary0(doEmit, type) \ + iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \ + (deadCode_ || (doEmit(), true)) + +#define dispatchTernary1(arg1, type) \ + iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \ + (deadCode_ || (emitTernary(arg1), true)) + +#define dispatchTernary2(arg1, type) \ + iter_.readTernary(type, &unused_a, &unused_b, &unused_c) && \ + (deadCode_ || (emitTernaryResultLast(arg1), true)) + +#define dispatchComparison0(doEmit, operandType, compareOp) \ + iter_.readComparison(operandType, &unused_a, &unused_b) && \ + (deadCode_ || (doEmit(compareOp, operandType), true)) + +#define dispatchConversion0(doEmit, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && \ + (deadCode_ || (doEmit(), true)) + +#define dispatchConversion1(arg1, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && \ + (deadCode_ || (emitUnop(arg1), true)) + +#define dispatchConversionOOM(doEmit, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || doEmit()) + +#define dispatchCalloutConversionOOM(doEmit, symbol, inType, outType) \ + iter_.readConversion(inType, outType, &unused_a) && \ + (deadCode_ || doEmit(symbol, inType, outType)) + +#define dispatchIntDivCallout(doEmit, symbol, type) \ + iter_.readBinary(type, &unused_a, &unused_b) && \ + (deadCode_ || doEmit(symbol, type)) + +#define dispatchVectorBinary(op) \ + iter_.readBinary(ValType::V128, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(op), true)) + +#define dispatchVectorUnary(op) \ + iter_.readUnary(ValType::V128, &unused_a) && \ + (deadCode_ || (emitUnop(op), true)) + +#define dispatchVectorComparison(op, compareOp) \ + iter_.readBinary(ValType::V128, &unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(compareOp, op), true)) + +#define dispatchVectorVariableShift(op) \ + iter_.readVectorShift(&unused_a, &unused_b) && \ + (deadCode_ || (emitBinop(op), true)) + +#define dispatchExtractLane(op, outType, laneLimit) \ + iter_.readExtractLane(outType, laneLimit, &laneIndex, &unused_a) && \ + (deadCode_ || (emitUnop(laneIndex, op), true)) + +#define dispatchReplaceLane(op, inType, laneLimit) \ + iter_.readReplaceLane(inType, laneLimit, &laneIndex, &unused_a, \ + &unused_b) && \ + (deadCode_ || (emitBinop(laneIndex, op), true)) + +#define dispatchSplat(op, inType) \ + iter_.readConversion(inType, ValType::V128, &unused_a) && \ + (deadCode_ || (emitUnop(op), true)) + +#define dispatchVectorReduction(op) \ + iter_.readConversion(ValType::V128, ValType::I32, &unused_a) && \ + (deadCode_ || (emitUnop(op), true)) + +#ifdef DEBUG + // Check that the number of ref-typed entries in the operand stack matches + // reality. +# define CHECK_POINTER_COUNT \ + do { \ + MOZ_ASSERT(countMemRefsOnStk() == stackMapGenerator_.memRefsOnStk); \ + } while (0) +#else +# define CHECK_POINTER_COUNT \ + do { \ + } while (0) +#endif + +#define CHECK(E) \ + if (!(E)) return false +#define NEXT() \ + { \ + CHECK_POINTER_COUNT; \ + continue; \ + } +#define CHECK_NEXT(E) \ + if (!(E)) return false; \ + { \ + CHECK_POINTER_COUNT; \ + continue; \ + } + + // Opcodes that push more than MaxPushesPerOpcode (anything with multiple + // results) will perform additional reservation. + CHECK(stk_.reserve(stk_.length() + MaxPushesPerOpcode)); + + OpBytes op{}; + CHECK(iter_.readOp(&op)); + + // When compilerEnv_.debugEnabled(), some operators get a breakpoint site. + if (compilerEnv_.debugEnabled() && op.shouldHaveBreakpoint()) { + if (previousBreakablePoint_ != masm.currentOffset()) { + // TODO sync only registers that can be clobbered by the exit + // prologue/epilogue or disable these registers for use in + // baseline compiler when compilerEnv_.debugEnabled() is set. + sync(); + + insertBreakablePoint(CallSiteDesc::Breakpoint); + if (!createStackMap("debug: per-insn breakpoint")) { + return false; + } + previousBreakablePoint_ = masm.currentOffset(); + } + } + + // Going below framePushedAtEntryToBody would imply that we've + // popped off the machine stack, part of the frame created by + // beginFunction(). + MOZ_ASSERT(masm.framePushed() >= + stackMapGenerator_.framePushedAtEntryToBody.value()); + + // At this point we're definitely not generating code for a function call. + MOZ_ASSERT( + stackMapGenerator_.framePushedExcludingOutboundCallArgs.isNothing()); + + switch (op.b0) { + case uint16_t(Op::End): + if (!emitEnd()) { + return false; + } + if (iter_.controlStackEmpty()) { + return true; + } + NEXT(); + + // Control opcodes + case uint16_t(Op::Nop): + CHECK_NEXT(iter_.readNop()); + case uint16_t(Op::Drop): + CHECK_NEXT(emitDrop()); + case uint16_t(Op::Block): + CHECK_NEXT(emitBlock()); + case uint16_t(Op::Loop): + CHECK_NEXT(emitLoop()); + case uint16_t(Op::If): + CHECK_NEXT(emitIf()); + case uint16_t(Op::Else): + CHECK_NEXT(emitElse()); + case uint16_t(Op::Try): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitTry()); + case uint16_t(Op::Catch): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitCatch()); + case uint16_t(Op::CatchAll): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitCatchAll()); + case uint16_t(Op::Delegate): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK(emitDelegate()); + iter_.popDelegate(); + NEXT(); + case uint16_t(Op::Throw): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitThrow()); + case uint16_t(Op::Rethrow): + if (!moduleEnv_.exceptionsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitRethrow()); + case uint16_t(Op::Br): + CHECK_NEXT(emitBr()); + case uint16_t(Op::BrIf): + CHECK_NEXT(emitBrIf()); + case uint16_t(Op::BrTable): + CHECK_NEXT(emitBrTable()); + case uint16_t(Op::Return): + CHECK_NEXT(emitReturn()); + case uint16_t(Op::Unreachable): + CHECK(iter_.readUnreachable()); + if (!deadCode_) { + trap(Trap::Unreachable); + deadCode_ = true; + } + NEXT(); + + // Calls + case uint16_t(Op::Call): + CHECK_NEXT(emitCall()); + case uint16_t(Op::CallIndirect): + CHECK_NEXT(emitCallIndirect()); +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::CallRef): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitCallRef()); +#endif + + // Locals and globals + case uint16_t(Op::LocalGet): + CHECK_NEXT(emitGetLocal()); + case uint16_t(Op::LocalSet): + CHECK_NEXT(emitSetLocal()); + case uint16_t(Op::LocalTee): + CHECK_NEXT(emitTeeLocal()); + case uint16_t(Op::GlobalGet): + CHECK_NEXT(emitGetGlobal()); + case uint16_t(Op::GlobalSet): + CHECK_NEXT(emitSetGlobal()); + case uint16_t(Op::TableGet): + CHECK_NEXT(emitTableGet()); + case uint16_t(Op::TableSet): + CHECK_NEXT(emitTableSet()); + + // Select + case uint16_t(Op::SelectNumeric): + CHECK_NEXT(emitSelect(/*typed*/ false)); + case uint16_t(Op::SelectTyped): + CHECK_NEXT(emitSelect(/*typed*/ true)); + + // I32 + case uint16_t(Op::I32Const): { + int32_t i32; + CHECK(iter_.readI32Const(&i32)); + if (!deadCode_) { + pushI32(i32); + } + NEXT(); + } + case uint16_t(Op::I32Add): + CHECK_NEXT(dispatchBinary2(AddI32, AddImmI32, ValType::I32)); + case uint16_t(Op::I32Sub): + CHECK_NEXT(dispatchBinary2(SubI32, SubImmI32, ValType::I32)); + case uint16_t(Op::I32Mul): + CHECK_NEXT(dispatchBinary1(MulI32, ValType::I32)); + case uint16_t(Op::I32DivS): + CHECK_NEXT(dispatchBinary0(emitQuotientI32, ValType::I32)); + case uint16_t(Op::I32DivU): + CHECK_NEXT(dispatchBinary0(emitQuotientU32, ValType::I32)); + case uint16_t(Op::I32RemS): + CHECK_NEXT(dispatchBinary0(emitRemainderI32, ValType::I32)); + case uint16_t(Op::I32RemU): + CHECK_NEXT(dispatchBinary0(emitRemainderU32, ValType::I32)); + case uint16_t(Op::I32Eqz): + CHECK_NEXT(dispatchConversion0(emitEqzI32, ValType::I32, ValType::I32)); + case uint16_t(Op::I32TruncF32S): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI32<0>, ValType::F32, + ValType::I32)); + case uint16_t(Op::I32TruncF32U): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI32, + ValType::F32, ValType::I32)); + case uint16_t(Op::I32TruncF64S): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI32<0>, ValType::F64, + ValType::I32)); + case uint16_t(Op::I32TruncF64U): + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI32, + ValType::F64, ValType::I32)); + case uint16_t(Op::I32WrapI64): + CHECK_NEXT( + dispatchConversion1(WrapI64ToI32, ValType::I64, ValType::I32)); + case uint16_t(Op::I32ReinterpretF32): + CHECK_NEXT(dispatchConversion1(ReinterpretF32AsI32, ValType::F32, + ValType::I32)); + case uint16_t(Op::I32Clz): + CHECK_NEXT(dispatchUnary1(ClzI32, ValType::I32)); + case uint16_t(Op::I32Ctz): + CHECK_NEXT(dispatchUnary1(CtzI32, ValType::I32)); + case uint16_t(Op::I32Popcnt): + CHECK_NEXT(dispatchUnary2(PopcntI32, PopcntTemp, ValType::I32)); + case uint16_t(Op::I32Or): + CHECK_NEXT(dispatchBinary2(OrI32, OrImmI32, ValType::I32)); + case uint16_t(Op::I32And): + CHECK_NEXT(dispatchBinary2(AndI32, AndImmI32, ValType::I32)); + case uint16_t(Op::I32Xor): + CHECK_NEXT(dispatchBinary2(XorI32, XorImmI32, ValType::I32)); + case uint16_t(Op::I32Shl): + CHECK_NEXT(dispatchBinary3( + ShlI32, ShlImmI32, &BaseCompiler::popI32RhsForShift, ValType::I32)); + case uint16_t(Op::I32ShrS): + CHECK_NEXT(dispatchBinary3( + ShrI32, ShrImmI32, &BaseCompiler::popI32RhsForShift, ValType::I32)); + case uint16_t(Op::I32ShrU): + CHECK_NEXT(dispatchBinary3(ShrUI32, ShrUImmI32, + &BaseCompiler::popI32RhsForShift, + ValType::I32)); + case uint16_t(Op::I32Load8S): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int8)); + case uint16_t(Op::I32Load8U): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint8)); + case uint16_t(Op::I32Load16S): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int16)); + case uint16_t(Op::I32Load16U): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint16)); + case uint16_t(Op::I32Load): + CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int32)); + case uint16_t(Op::I32Store8): + CHECK_NEXT(emitStore(ValType::I32, Scalar::Int8)); + case uint16_t(Op::I32Store16): + CHECK_NEXT(emitStore(ValType::I32, Scalar::Int16)); + case uint16_t(Op::I32Store): + CHECK_NEXT(emitStore(ValType::I32, Scalar::Int32)); + case uint16_t(Op::I32Rotr): + CHECK_NEXT(dispatchBinary3(RotrI32, RotrImmI32, + &BaseCompiler::popI32RhsForRotate, + ValType::I32)); + case uint16_t(Op::I32Rotl): + CHECK_NEXT(dispatchBinary3(RotlI32, RotlImmI32, + &BaseCompiler::popI32RhsForRotate, + ValType::I32)); + + // I64 + case uint16_t(Op::I64Const): { + int64_t i64; + CHECK(iter_.readI64Const(&i64)); + if (!deadCode_) { + pushI64(i64); + } + NEXT(); + } + case uint16_t(Op::I64Add): + CHECK_NEXT(dispatchBinary2(AddI64, AddImmI64, ValType::I64)); + case uint16_t(Op::I64Sub): + CHECK_NEXT(dispatchBinary2(SubI64, SubImmI64, ValType::I64)); + case uint16_t(Op::I64Mul): + CHECK_NEXT(dispatchBinary0(emitMultiplyI64, ValType::I64)); + case uint16_t(Op::I64DivS): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout( + emitDivOrModI64BuiltinCall, SymbolicAddress::DivI64, ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitQuotientI64, ValType::I64)); +#endif + case uint16_t(Op::I64DivU): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout(emitDivOrModI64BuiltinCall, + SymbolicAddress::UDivI64, + ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitQuotientU64, ValType::I64)); +#endif + case uint16_t(Op::I64RemS): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout( + emitDivOrModI64BuiltinCall, SymbolicAddress::ModI64, ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitRemainderI64, ValType::I64)); +#endif + case uint16_t(Op::I64RemU): +#ifdef RABALDR_INT_DIV_I64_CALLOUT + CHECK_NEXT(dispatchIntDivCallout(emitDivOrModI64BuiltinCall, + SymbolicAddress::UModI64, + ValType::I64)); +#else + CHECK_NEXT(dispatchBinary0(emitRemainderU64, ValType::I64)); +#endif + case uint16_t(Op::I64TruncF32S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT( + dispatchCalloutConversionOOM(emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToInt64, + ValType::F32, ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI64<0>, ValType::F32, + ValType::I64)); +#endif + case uint16_t(Op::I64TruncF32U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToUint64, ValType::F32, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF32ToI64, + ValType::F32, ValType::I64)); +#endif + case uint16_t(Op::I64TruncF64S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT( + dispatchCalloutConversionOOM(emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToInt64, + ValType::F64, ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI64<0>, ValType::F64, + ValType::I64)); +#endif + case uint16_t(Op::I64TruncF64U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::TruncateDoubleToUint64, ValType::F64, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM(emitTruncateF64ToI64, + ValType::F64, ValType::I64)); +#endif + case uint16_t(Op::I64ExtendI32S): + CHECK_NEXT(dispatchConversion0(emitExtendI32ToI64, ValType::I32, + ValType::I64)); + case uint16_t(Op::I64ExtendI32U): + CHECK_NEXT(dispatchConversion0(emitExtendU32ToI64, ValType::I32, + ValType::I64)); + case uint16_t(Op::I64ReinterpretF64): + CHECK_NEXT(dispatchConversion1(ReinterpretF64AsI64, ValType::F64, + ValType::I64)); + case uint16_t(Op::I64Or): + CHECK_NEXT(dispatchBinary2(OrI64, OrImmI64, ValType::I64)); + case uint16_t(Op::I64And): + CHECK_NEXT(dispatchBinary2(AndI64, AndImmI64, ValType::I64)); + case uint16_t(Op::I64Xor): + CHECK_NEXT(dispatchBinary2(XorI64, XorImmI64, ValType::I64)); + case uint16_t(Op::I64Shl): + CHECK_NEXT(dispatchBinary3( + ShlI64, ShlImmI64, &BaseCompiler::popI64RhsForShift, ValType::I64)); + case uint16_t(Op::I64ShrS): + CHECK_NEXT(dispatchBinary3( + ShrI64, ShrImmI64, &BaseCompiler::popI64RhsForShift, ValType::I64)); + case uint16_t(Op::I64ShrU): + CHECK_NEXT(dispatchBinary3(ShrUI64, ShrUImmI64, + &BaseCompiler::popI64RhsForShift, + ValType::I64)); + case uint16_t(Op::I64Rotr): + CHECK_NEXT(dispatchBinary0(emitRotrI64, ValType::I64)); + case uint16_t(Op::I64Rotl): + CHECK_NEXT(dispatchBinary0(emitRotlI64, ValType::I64)); + case uint16_t(Op::I64Clz): + CHECK_NEXT(dispatchUnary1(ClzI64, ValType::I64)); + case uint16_t(Op::I64Ctz): + CHECK_NEXT(dispatchUnary1(CtzI64, ValType::I64)); + case uint16_t(Op::I64Popcnt): + CHECK_NEXT(dispatchUnary2(PopcntI64, PopcntTemp, ValType::I64)); + case uint16_t(Op::I64Eqz): + CHECK_NEXT(dispatchConversion0(emitEqzI64, ValType::I64, ValType::I32)); + case uint16_t(Op::I64Load8S): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int8)); + case uint16_t(Op::I64Load16S): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int16)); + case uint16_t(Op::I64Load32S): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int32)); + case uint16_t(Op::I64Load8U): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint8)); + case uint16_t(Op::I64Load16U): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint16)); + case uint16_t(Op::I64Load32U): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint32)); + case uint16_t(Op::I64Load): + CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int64)); + case uint16_t(Op::I64Store8): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int8)); + case uint16_t(Op::I64Store16): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int16)); + case uint16_t(Op::I64Store32): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int32)); + case uint16_t(Op::I64Store): + CHECK_NEXT(emitStore(ValType::I64, Scalar::Int64)); + + // F32 + case uint16_t(Op::F32Const): { + float f32; + CHECK(iter_.readF32Const(&f32)); + if (!deadCode_) { + pushF32(f32); + } + NEXT(); + } + case uint16_t(Op::F32Add): + CHECK_NEXT(dispatchBinary1(AddF32, ValType::F32)) + case uint16_t(Op::F32Sub): + CHECK_NEXT(dispatchBinary1(SubF32, ValType::F32)); + case uint16_t(Op::F32Mul): + CHECK_NEXT(dispatchBinary1(MulF32, ValType::F32)); + case uint16_t(Op::F32Div): + CHECK_NEXT(dispatchBinary1(DivF32, ValType::F32)); + case uint16_t(Op::F32Min): + CHECK_NEXT(dispatchBinary1(MinF32, ValType::F32)); + case uint16_t(Op::F32Max): + CHECK_NEXT(dispatchBinary1(MaxF32, ValType::F32)); + case uint16_t(Op::F32Neg): + CHECK_NEXT(dispatchUnary1(NegateF32, ValType::F32)); + case uint16_t(Op::F32Abs): + CHECK_NEXT(dispatchUnary1(AbsF32, ValType::F32)); + case uint16_t(Op::F32Sqrt): + CHECK_NEXT(dispatchUnary1(SqrtF32, ValType::F32)); + case uint16_t(Op::F32Ceil): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::CeilF, ValType::F32)); + case uint16_t(Op::F32Floor): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::FloorF, ValType::F32)); + case uint16_t(Op::F32DemoteF64): + CHECK_NEXT( + dispatchConversion1(ConvertF64ToF32, ValType::F64, ValType::F32)); + case uint16_t(Op::F32ConvertI32S): + CHECK_NEXT( + dispatchConversion1(ConvertI32ToF32, ValType::I32, ValType::F32)); + case uint16_t(Op::F32ConvertI32U): + CHECK_NEXT( + dispatchConversion1(ConvertU32ToF32, ValType::I32, ValType::F32)); + case uint16_t(Op::F32ConvertI64S): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Int64ToFloat32, + ValType::I64, ValType::F32)); +#else + CHECK_NEXT( + dispatchConversion1(ConvertI64ToF32, ValType::I64, ValType::F32)); +#endif + case uint16_t(Op::F32ConvertI64U): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Uint64ToFloat32, + ValType::I64, ValType::F32)); +#else + CHECK_NEXT(dispatchConversion0(emitConvertU64ToF32, ValType::I64, + ValType::F32)); +#endif + case uint16_t(Op::F32ReinterpretI32): + CHECK_NEXT(dispatchConversion1(ReinterpretI32AsF32, ValType::I32, + ValType::F32)); + case uint16_t(Op::F32Load): + CHECK_NEXT(emitLoad(ValType::F32, Scalar::Float32)); + case uint16_t(Op::F32Store): + CHECK_NEXT(emitStore(ValType::F32, Scalar::Float32)); + case uint16_t(Op::F32CopySign): + CHECK_NEXT(dispatchBinary1(CopysignF32, ValType::F32)); + case uint16_t(Op::F32Nearest): + CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntF, + ValType::F32)); + case uint16_t(Op::F32Trunc): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::TruncF, ValType::F32)); + + // F64 + case uint16_t(Op::F64Const): { + double f64; + CHECK(iter_.readF64Const(&f64)); + if (!deadCode_) { + pushF64(f64); + } + NEXT(); + } + case uint16_t(Op::F64Add): + CHECK_NEXT(dispatchBinary1(AddF64, ValType::F64)) + case uint16_t(Op::F64Sub): + CHECK_NEXT(dispatchBinary1(SubF64, ValType::F64)); + case uint16_t(Op::F64Mul): + CHECK_NEXT(dispatchBinary1(MulF64, ValType::F64)); + case uint16_t(Op::F64Div): + CHECK_NEXT(dispatchBinary1(DivF64, ValType::F64)); + case uint16_t(Op::F64Min): + CHECK_NEXT(dispatchBinary1(MinF64, ValType::F64)); + case uint16_t(Op::F64Max): + CHECK_NEXT(dispatchBinary1(MaxF64, ValType::F64)); + case uint16_t(Op::F64Neg): + CHECK_NEXT(dispatchUnary1(NegateF64, ValType::F64)); + case uint16_t(Op::F64Abs): + CHECK_NEXT(dispatchUnary1(AbsF64, ValType::F64)); + case uint16_t(Op::F64Sqrt): + CHECK_NEXT(dispatchUnary1(SqrtF64, ValType::F64)); + case uint16_t(Op::F64Ceil): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::CeilD, ValType::F64)); + case uint16_t(Op::F64Floor): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::FloorD, ValType::F64)); + case uint16_t(Op::F64PromoteF32): + CHECK_NEXT( + dispatchConversion1(ConvertF32ToF64, ValType::F32, ValType::F64)); + case uint16_t(Op::F64ConvertI32S): + CHECK_NEXT( + dispatchConversion1(ConvertI32ToF64, ValType::I32, ValType::F64)); + case uint16_t(Op::F64ConvertI32U): + CHECK_NEXT( + dispatchConversion1(ConvertU32ToF64, ValType::I32, ValType::F64)); + case uint16_t(Op::F64ConvertI64S): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Int64ToDouble, + ValType::I64, ValType::F64)); +#else + CHECK_NEXT( + dispatchConversion1(ConvertI64ToF64, ValType::I64, ValType::F64)); +#endif + case uint16_t(Op::F64ConvertI64U): +#ifdef RABALDR_I64_TO_FLOAT_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertInt64ToFloatingCallout, SymbolicAddress::Uint64ToDouble, + ValType::I64, ValType::F64)); +#else + CHECK_NEXT(dispatchConversion0(emitConvertU64ToF64, ValType::I64, + ValType::F64)); +#endif + case uint16_t(Op::F64Load): + CHECK_NEXT(emitLoad(ValType::F64, Scalar::Float64)); + case uint16_t(Op::F64Store): + CHECK_NEXT(emitStore(ValType::F64, Scalar::Float64)); + case uint16_t(Op::F64ReinterpretI64): + CHECK_NEXT(dispatchConversion1(ReinterpretI64AsF64, ValType::I64, + ValType::F64)); + case uint16_t(Op::F64CopySign): + CHECK_NEXT(dispatchBinary1(CopysignF64, ValType::F64)); + case uint16_t(Op::F64Nearest): + CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntD, + ValType::F64)); + case uint16_t(Op::F64Trunc): + CHECK_NEXT( + emitUnaryMathBuiltinCall(SymbolicAddress::TruncD, ValType::F64)); + + // Comparisons + case uint16_t(Op::I32Eq): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::Equal)); + case uint16_t(Op::I32Ne): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::NotEqual)); + case uint16_t(Op::I32LtS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::LessThan)); + case uint16_t(Op::I32LeS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::LessThanOrEqual)); + case uint16_t(Op::I32GtS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::GreaterThan)); + case uint16_t(Op::I32GeS): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::GreaterThanOrEqual)); + case uint16_t(Op::I32LtU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::Below)); + case uint16_t(Op::I32LeU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::BelowOrEqual)); + case uint16_t(Op::I32GtU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::Above)); + case uint16_t(Op::I32GeU): + CHECK_NEXT(dispatchComparison0(emitCompareI32, ValType::I32, + Assembler::AboveOrEqual)); + case uint16_t(Op::I64Eq): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::Equal)); + case uint16_t(Op::I64Ne): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::NotEqual)); + case uint16_t(Op::I64LtS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::LessThan)); + case uint16_t(Op::I64LeS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::LessThanOrEqual)); + case uint16_t(Op::I64GtS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::GreaterThan)); + case uint16_t(Op::I64GeS): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::GreaterThanOrEqual)); + case uint16_t(Op::I64LtU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::Below)); + case uint16_t(Op::I64LeU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::BelowOrEqual)); + case uint16_t(Op::I64GtU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::Above)); + case uint16_t(Op::I64GeU): + CHECK_NEXT(dispatchComparison0(emitCompareI64, ValType::I64, + Assembler::AboveOrEqual)); + case uint16_t(Op::F32Eq): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleEqual)); + case uint16_t(Op::F32Ne): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleNotEqualOrUnordered)); + case uint16_t(Op::F32Lt): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleLessThan)); + case uint16_t(Op::F32Le): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleLessThanOrEqual)); + case uint16_t(Op::F32Gt): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleGreaterThan)); + case uint16_t(Op::F32Ge): + CHECK_NEXT(dispatchComparison0(emitCompareF32, ValType::F32, + Assembler::DoubleGreaterThanOrEqual)); + case uint16_t(Op::F64Eq): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleEqual)); + case uint16_t(Op::F64Ne): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleNotEqualOrUnordered)); + case uint16_t(Op::F64Lt): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleLessThan)); + case uint16_t(Op::F64Le): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleLessThanOrEqual)); + case uint16_t(Op::F64Gt): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleGreaterThan)); + case uint16_t(Op::F64Ge): + CHECK_NEXT(dispatchComparison0(emitCompareF64, ValType::F64, + Assembler::DoubleGreaterThanOrEqual)); + + // Sign extensions + case uint16_t(Op::I32Extend8S): + CHECK_NEXT( + dispatchConversion1(ExtendI32_8, ValType::I32, ValType::I32)); + case uint16_t(Op::I32Extend16S): + CHECK_NEXT( + dispatchConversion1(ExtendI32_16, ValType::I32, ValType::I32)); + case uint16_t(Op::I64Extend8S): + CHECK_NEXT( + dispatchConversion0(emitExtendI64_8, ValType::I64, ValType::I64)); + case uint16_t(Op::I64Extend16S): + CHECK_NEXT( + dispatchConversion0(emitExtendI64_16, ValType::I64, ValType::I64)); + case uint16_t(Op::I64Extend32S): + CHECK_NEXT( + dispatchConversion0(emitExtendI64_32, ValType::I64, ValType::I64)); + + // Memory Related + case uint16_t(Op::MemoryGrow): + CHECK_NEXT(emitMemoryGrow()); + case uint16_t(Op::MemorySize): + CHECK_NEXT(emitMemorySize()); + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::RefAsNonNull): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitRefAsNonNull()); + case uint16_t(Op::BrOnNull): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitBrOnNull()); + case uint16_t(Op::BrOnNonNull): + if (!moduleEnv_.functionReferencesEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitBrOnNonNull()); +#endif +#ifdef ENABLE_WASM_GC + case uint16_t(Op::RefEq): + if (!moduleEnv_.gcEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchComparison0(emitCompareRef, RefType::eq(), + Assembler::Equal)); +#endif + case uint16_t(Op::RefFunc): + CHECK_NEXT(emitRefFunc()); + break; + case uint16_t(Op::RefNull): + CHECK_NEXT(emitRefNull()); + break; + case uint16_t(Op::RefIsNull): + CHECK_NEXT(emitRefIsNull()); + break; + +#ifdef ENABLE_WASM_GC + // "GC" operations + case uint16_t(Op::GcPrefix): { + if (!moduleEnv_.gcEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(GcOp::StructNew): + CHECK_NEXT(emitStructNew()); + case uint32_t(GcOp::StructNewDefault): + CHECK_NEXT(emitStructNewDefault()); + case uint32_t(GcOp::StructGet): + CHECK_NEXT(emitStructGet(FieldWideningOp::None)); + case uint32_t(GcOp::StructGetS): + CHECK_NEXT(emitStructGet(FieldWideningOp::Signed)); + case uint32_t(GcOp::StructGetU): + CHECK_NEXT(emitStructGet(FieldWideningOp::Unsigned)); + case uint32_t(GcOp::StructSet): + CHECK_NEXT(emitStructSet()); + case uint32_t(GcOp::ArrayNew): + CHECK_NEXT(emitArrayNew()); + case uint32_t(GcOp::ArrayNewFixed): + CHECK_NEXT(emitArrayNewFixed()); + case uint32_t(GcOp::ArrayNewDefault): + CHECK_NEXT(emitArrayNewDefault()); + case uint32_t(GcOp::ArrayNewData): + CHECK_NEXT(emitArrayNewData()); + case uint32_t(GcOp::ArrayInitFromElemStaticV5): + case uint32_t(GcOp::ArrayNewElem): + CHECK_NEXT(emitArrayNewElem()); + case uint32_t(GcOp::ArrayGet): + CHECK_NEXT(emitArrayGet(FieldWideningOp::None)); + case uint32_t(GcOp::ArrayGetS): + CHECK_NEXT(emitArrayGet(FieldWideningOp::Signed)); + case uint32_t(GcOp::ArrayGetU): + CHECK_NEXT(emitArrayGet(FieldWideningOp::Unsigned)); + case uint32_t(GcOp::ArraySet): + CHECK_NEXT(emitArraySet()); + case uint32_t(GcOp::ArrayLenWithTypeIndex): + CHECK_NEXT(emitArrayLen(/*decodeIgnoredTypeIndex=*/true)); + case uint32_t(GcOp::ArrayLen): + CHECK_NEXT(emitArrayLen(/*decodeIgnoredTypeIndex=*/false)); + case uint32_t(GcOp::ArrayCopy): + CHECK_NEXT(emitArrayCopy()); + case uint32_t(GcOp::RefTestV5): + CHECK_NEXT(emitRefTestV5()); + case uint32_t(GcOp::RefCastV5): + CHECK_NEXT(emitRefCastV5()); + case uint32_t(GcOp::RefTest): + CHECK_NEXT(emitRefTest(/*nullable=*/false)); + case uint32_t(GcOp::RefTestNull): + CHECK_NEXT(emitRefTest(/*nullable=*/true)); + case uint32_t(GcOp::RefCast): + CHECK_NEXT(emitRefCast(/*nullable=*/false)); + case uint32_t(GcOp::RefCastNull): + CHECK_NEXT(emitRefCast(/*nullable=*/true)); + case uint32_t(GcOp::BrOnCast): + CHECK_NEXT(emitBrOnCast()); + case uint32_t(GcOp::BrOnCastV5): + CHECK_NEXT(emitBrOnCastV5(/*onSuccess=*/true)); + case uint32_t(GcOp::BrOnCastFailV5): + CHECK_NEXT(emitBrOnCastV5(/*onSuccess=*/false)); + case uint32_t(GcOp::BrOnCastHeapV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/true, /*nullable=*/false)); + case uint32_t(GcOp::BrOnCastHeapNullV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/true, /*nullable=*/true)); + case uint32_t(GcOp::BrOnCastFailHeapV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/false, /*nullable=*/false)); + case uint32_t(GcOp::BrOnCastFailHeapNullV5): + CHECK_NEXT( + emitBrOnCastHeapV5(/*onSuccess=*/false, /*nullable=*/true)); + case uint32_t(GcOp::RefAsStructV5): + CHECK_NEXT(emitRefAsStructV5()); + case uint32_t(GcOp::BrOnNonStructV5): + CHECK_NEXT(emitBrOnNonStructV5()); + case uint16_t(GcOp::ExternInternalize): + CHECK_NEXT(emitExternInternalize()); + case uint16_t(GcOp::ExternExternalize): + CHECK_NEXT(emitExternExternalize()); + default: + break; + } // switch (op.b1) + return iter_.unrecognizedOpcode(&op); + } +#endif + +#ifdef ENABLE_WASM_SIMD + // SIMD operations + case uint16_t(Op::SimdPrefix): { + uint32_t laneIndex; + if (!moduleEnv_.simdAvailable()) { + return iter_.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(SimdOp::I8x16ExtractLaneS): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI8x16, ValType::I32, 16)); + case uint32_t(SimdOp::I8x16ExtractLaneU): + CHECK_NEXT( + dispatchExtractLane(ExtractLaneUI8x16, ValType::I32, 16)); + case uint32_t(SimdOp::I16x8ExtractLaneS): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI16x8, ValType::I32, 8)); + case uint32_t(SimdOp::I16x8ExtractLaneU): + CHECK_NEXT(dispatchExtractLane(ExtractLaneUI16x8, ValType::I32, 8)); + case uint32_t(SimdOp::I32x4ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI32x4, ValType::I32, 4)); + case uint32_t(SimdOp::I64x2ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneI64x2, ValType::I64, 2)); + case uint32_t(SimdOp::F32x4ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneF32x4, ValType::F32, 4)); + case uint32_t(SimdOp::F64x2ExtractLane): + CHECK_NEXT(dispatchExtractLane(ExtractLaneF64x2, ValType::F64, 2)); + case uint32_t(SimdOp::I8x16Splat): + CHECK_NEXT(dispatchSplat(SplatI8x16, ValType::I32)); + case uint32_t(SimdOp::I16x8Splat): + CHECK_NEXT(dispatchSplat(SplatI16x8, ValType::I32)); + case uint32_t(SimdOp::I32x4Splat): + CHECK_NEXT(dispatchSplat(SplatI32x4, ValType::I32)); + case uint32_t(SimdOp::I64x2Splat): + CHECK_NEXT(dispatchSplat(SplatI64x2, ValType::I64)); + case uint32_t(SimdOp::F32x4Splat): + CHECK_NEXT(dispatchSplat(SplatF32x4, ValType::F32)); + case uint32_t(SimdOp::F64x2Splat): + CHECK_NEXT(dispatchSplat(SplatF64x2, ValType::F64)); + case uint32_t(SimdOp::V128AnyTrue): + CHECK_NEXT(dispatchVectorReduction(AnyTrue)); + case uint32_t(SimdOp::I8x16AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI8x16)); + case uint32_t(SimdOp::I16x8AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI16x8)); + case uint32_t(SimdOp::I32x4AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI32x4)); + case uint32_t(SimdOp::I64x2AllTrue): + CHECK_NEXT(dispatchVectorReduction(AllTrueI64x2)); + case uint32_t(SimdOp::I8x16Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI8x16)); + case uint32_t(SimdOp::I16x8Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI16x8)); + case uint32_t(SimdOp::I32x4Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI32x4)); + case uint32_t(SimdOp::I64x2Bitmask): + CHECK_NEXT(dispatchVectorReduction(BitmaskI64x2)); + case uint32_t(SimdOp::I8x16ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI8x16, ValType::I32, 16)); + case uint32_t(SimdOp::I16x8ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI16x8, ValType::I32, 8)); + case uint32_t(SimdOp::I32x4ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI32x4, ValType::I32, 4)); + case uint32_t(SimdOp::I64x2ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneI64x2, ValType::I64, 2)); + case uint32_t(SimdOp::F32x4ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneF32x4, ValType::F32, 4)); + case uint32_t(SimdOp::F64x2ReplaceLane): + CHECK_NEXT(dispatchReplaceLane(ReplaceLaneF64x2, ValType::F64, 2)); + case uint32_t(SimdOp::I8x16Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::Equal)); + case uint32_t(SimdOp::I8x16Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::NotEqual)); + case uint32_t(SimdOp::I8x16LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, Assembler::LessThan)); + case uint32_t(SimdOp::I8x16LtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI8x16, Assembler::Below)); + case uint32_t(SimdOp::I8x16GtS): + CHECK_NEXT( + dispatchVectorComparison(CmpI8x16, Assembler::GreaterThan)); + case uint32_t(SimdOp::I8x16GtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI8x16, Assembler::Above)); + case uint32_t(SimdOp::I8x16LeS): + CHECK_NEXT( + dispatchVectorComparison(CmpI8x16, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I8x16LeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI8x16, Assembler::BelowOrEqual)); + case uint32_t(SimdOp::I8x16GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI8x16, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::I8x16GeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI8x16, Assembler::AboveOrEqual)); + case uint32_t(SimdOp::I16x8Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::Equal)); + case uint32_t(SimdOp::I16x8Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::NotEqual)); + case uint32_t(SimdOp::I16x8LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, Assembler::LessThan)); + case uint32_t(SimdOp::I16x8LtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI16x8, Assembler::Below)); + case uint32_t(SimdOp::I16x8GtS): + CHECK_NEXT( + dispatchVectorComparison(CmpI16x8, Assembler::GreaterThan)); + case uint32_t(SimdOp::I16x8GtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI16x8, Assembler::Above)); + case uint32_t(SimdOp::I16x8LeS): + CHECK_NEXT( + dispatchVectorComparison(CmpI16x8, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I16x8LeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI16x8, Assembler::BelowOrEqual)); + case uint32_t(SimdOp::I16x8GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI16x8, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::I16x8GeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI16x8, Assembler::AboveOrEqual)); + case uint32_t(SimdOp::I32x4Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::Equal)); + case uint32_t(SimdOp::I32x4Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::NotEqual)); + case uint32_t(SimdOp::I32x4LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, Assembler::LessThan)); + case uint32_t(SimdOp::I32x4LtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI32x4, Assembler::Below)); + case uint32_t(SimdOp::I32x4GtS): + CHECK_NEXT( + dispatchVectorComparison(CmpI32x4, Assembler::GreaterThan)); + case uint32_t(SimdOp::I32x4GtU): + CHECK_NEXT(dispatchVectorComparison(CmpUI32x4, Assembler::Above)); + case uint32_t(SimdOp::I32x4LeS): + CHECK_NEXT( + dispatchVectorComparison(CmpI32x4, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I32x4LeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI32x4, Assembler::BelowOrEqual)); + case uint32_t(SimdOp::I32x4GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI32x4, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::I32x4GeU): + CHECK_NEXT( + dispatchVectorComparison(CmpUI32x4, Assembler::AboveOrEqual)); + case uint32_t(SimdOp::I64x2Eq): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForEquality, + Assembler::Equal)); + case uint32_t(SimdOp::I64x2Ne): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForEquality, + Assembler::NotEqual)); + case uint32_t(SimdOp::I64x2LtS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::LessThan)); + case uint32_t(SimdOp::I64x2GtS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::GreaterThan)); + case uint32_t(SimdOp::I64x2LeS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::I64x2GeS): + CHECK_NEXT(dispatchVectorComparison(CmpI64x2ForOrdering, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::F32x4Eq): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::Equal)); + case uint32_t(SimdOp::F32x4Ne): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::NotEqual)); + case uint32_t(SimdOp::F32x4Lt): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, Assembler::LessThan)); + case uint32_t(SimdOp::F32x4Gt): + CHECK_NEXT( + dispatchVectorComparison(CmpF32x4, Assembler::GreaterThan)); + case uint32_t(SimdOp::F32x4Le): + CHECK_NEXT( + dispatchVectorComparison(CmpF32x4, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::F32x4Ge): + CHECK_NEXT(dispatchVectorComparison(CmpF32x4, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::F64x2Eq): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::Equal)); + case uint32_t(SimdOp::F64x2Ne): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::NotEqual)); + case uint32_t(SimdOp::F64x2Lt): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, Assembler::LessThan)); + case uint32_t(SimdOp::F64x2Gt): + CHECK_NEXT( + dispatchVectorComparison(CmpF64x2, Assembler::GreaterThan)); + case uint32_t(SimdOp::F64x2Le): + CHECK_NEXT( + dispatchVectorComparison(CmpF64x2, Assembler::LessThanOrEqual)); + case uint32_t(SimdOp::F64x2Ge): + CHECK_NEXT(dispatchVectorComparison(CmpF64x2, + Assembler::GreaterThanOrEqual)); + case uint32_t(SimdOp::V128And): + CHECK_NEXT(dispatchVectorBinary(AndV128)); + case uint32_t(SimdOp::V128Or): + CHECK_NEXT(dispatchVectorBinary(OrV128)); + case uint32_t(SimdOp::V128Xor): + CHECK_NEXT(dispatchVectorBinary(XorV128)); + case uint32_t(SimdOp::V128AndNot): + CHECK_NEXT(dispatchBinary0(emitVectorAndNot, ValType::V128)); + case uint32_t(SimdOp::I8x16AvgrU): + CHECK_NEXT(dispatchVectorBinary(AverageUI8x16)); + case uint32_t(SimdOp::I16x8AvgrU): + CHECK_NEXT(dispatchVectorBinary(AverageUI16x8)); + case uint32_t(SimdOp::I8x16Add): + CHECK_NEXT(dispatchVectorBinary(AddI8x16)); + case uint32_t(SimdOp::I8x16AddSatS): + CHECK_NEXT(dispatchVectorBinary(AddSatI8x16)); + case uint32_t(SimdOp::I8x16AddSatU): + CHECK_NEXT(dispatchVectorBinary(AddSatUI8x16)); + case uint32_t(SimdOp::I8x16Sub): + CHECK_NEXT(dispatchVectorBinary(SubI8x16)); + case uint32_t(SimdOp::I8x16SubSatS): + CHECK_NEXT(dispatchVectorBinary(SubSatI8x16)); + case uint32_t(SimdOp::I8x16SubSatU): + CHECK_NEXT(dispatchVectorBinary(SubSatUI8x16)); + case uint32_t(SimdOp::I8x16MinS): + CHECK_NEXT(dispatchVectorBinary(MinI8x16)); + case uint32_t(SimdOp::I8x16MinU): + CHECK_NEXT(dispatchVectorBinary(MinUI8x16)); + case uint32_t(SimdOp::I8x16MaxS): + CHECK_NEXT(dispatchVectorBinary(MaxI8x16)); + case uint32_t(SimdOp::I8x16MaxU): + CHECK_NEXT(dispatchVectorBinary(MaxUI8x16)); + case uint32_t(SimdOp::I16x8Add): + CHECK_NEXT(dispatchVectorBinary(AddI16x8)); + case uint32_t(SimdOp::I16x8AddSatS): + CHECK_NEXT(dispatchVectorBinary(AddSatI16x8)); + case uint32_t(SimdOp::I16x8AddSatU): + CHECK_NEXT(dispatchVectorBinary(AddSatUI16x8)); + case uint32_t(SimdOp::I16x8Sub): + CHECK_NEXT(dispatchVectorBinary(SubI16x8)); + case uint32_t(SimdOp::I16x8SubSatS): + CHECK_NEXT(dispatchVectorBinary(SubSatI16x8)); + case uint32_t(SimdOp::I16x8SubSatU): + CHECK_NEXT(dispatchVectorBinary(SubSatUI16x8)); + case uint32_t(SimdOp::I16x8Mul): + CHECK_NEXT(dispatchVectorBinary(MulI16x8)); + case uint32_t(SimdOp::I16x8MinS): + CHECK_NEXT(dispatchVectorBinary(MinI16x8)); + case uint32_t(SimdOp::I16x8MinU): + CHECK_NEXT(dispatchVectorBinary(MinUI16x8)); + case uint32_t(SimdOp::I16x8MaxS): + CHECK_NEXT(dispatchVectorBinary(MaxI16x8)); + case uint32_t(SimdOp::I16x8MaxU): + CHECK_NEXT(dispatchVectorBinary(MaxUI16x8)); + case uint32_t(SimdOp::I32x4Add): + CHECK_NEXT(dispatchVectorBinary(AddI32x4)); + case uint32_t(SimdOp::I32x4Sub): + CHECK_NEXT(dispatchVectorBinary(SubI32x4)); + case uint32_t(SimdOp::I32x4Mul): + CHECK_NEXT(dispatchVectorBinary(MulI32x4)); + case uint32_t(SimdOp::I32x4MinS): + CHECK_NEXT(dispatchVectorBinary(MinI32x4)); + case uint32_t(SimdOp::I32x4MinU): + CHECK_NEXT(dispatchVectorBinary(MinUI32x4)); + case uint32_t(SimdOp::I32x4MaxS): + CHECK_NEXT(dispatchVectorBinary(MaxI32x4)); + case uint32_t(SimdOp::I32x4MaxU): + CHECK_NEXT(dispatchVectorBinary(MaxUI32x4)); + case uint32_t(SimdOp::I64x2Add): + CHECK_NEXT(dispatchVectorBinary(AddI64x2)); + case uint32_t(SimdOp::I64x2Sub): + CHECK_NEXT(dispatchVectorBinary(SubI64x2)); + case uint32_t(SimdOp::I64x2Mul): + CHECK_NEXT(dispatchVectorBinary(MulI64x2)); + case uint32_t(SimdOp::F32x4Add): + CHECK_NEXT(dispatchVectorBinary(AddF32x4)); + case uint32_t(SimdOp::F32x4Sub): + CHECK_NEXT(dispatchVectorBinary(SubF32x4)); + case uint32_t(SimdOp::F32x4Mul): + CHECK_NEXT(dispatchVectorBinary(MulF32x4)); + case uint32_t(SimdOp::F32x4Div): + CHECK_NEXT(dispatchVectorBinary(DivF32x4)); + case uint32_t(SimdOp::F32x4Min): + CHECK_NEXT(dispatchVectorBinary(MinF32x4)); + case uint32_t(SimdOp::F32x4Max): + CHECK_NEXT(dispatchVectorBinary(MaxF32x4)); + case uint32_t(SimdOp::F64x2Add): + CHECK_NEXT(dispatchVectorBinary(AddF64x2)); + case uint32_t(SimdOp::F64x2Sub): + CHECK_NEXT(dispatchVectorBinary(SubF64x2)); + case uint32_t(SimdOp::F64x2Mul): + CHECK_NEXT(dispatchVectorBinary(MulF64x2)); + case uint32_t(SimdOp::F64x2Div): + CHECK_NEXT(dispatchVectorBinary(DivF64x2)); + case uint32_t(SimdOp::F64x2Min): + CHECK_NEXT(dispatchVectorBinary(MinF64x2)); + case uint32_t(SimdOp::F64x2Max): + CHECK_NEXT(dispatchVectorBinary(MaxF64x2)); + case uint32_t(SimdOp::I8x16NarrowI16x8S): + CHECK_NEXT(dispatchVectorBinary(NarrowI16x8)); + case uint32_t(SimdOp::I8x16NarrowI16x8U): + CHECK_NEXT(dispatchVectorBinary(NarrowUI16x8)); + case uint32_t(SimdOp::I16x8NarrowI32x4S): + CHECK_NEXT(dispatchVectorBinary(NarrowI32x4)); + case uint32_t(SimdOp::I16x8NarrowI32x4U): + CHECK_NEXT(dispatchVectorBinary(NarrowUI32x4)); + case uint32_t(SimdOp::I8x16Swizzle): + CHECK_NEXT(dispatchVectorBinary(Swizzle)); + case uint32_t(SimdOp::F32x4PMax): + CHECK_NEXT(dispatchVectorBinary(PMaxF32x4)); + case uint32_t(SimdOp::F32x4PMin): + CHECK_NEXT(dispatchVectorBinary(PMinF32x4)); + case uint32_t(SimdOp::F64x2PMax): + CHECK_NEXT(dispatchVectorBinary(PMaxF64x2)); + case uint32_t(SimdOp::F64x2PMin): + CHECK_NEXT(dispatchVectorBinary(PMinF64x2)); + case uint32_t(SimdOp::I32x4DotI16x8S): + CHECK_NEXT(dispatchVectorBinary(DotI16x8)); + case uint32_t(SimdOp::I16x8ExtmulLowI8x16S): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowI8x16)); + case uint32_t(SimdOp::I16x8ExtmulHighI8x16S): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighI8x16)); + case uint32_t(SimdOp::I16x8ExtmulLowI8x16U): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI8x16)); + case uint32_t(SimdOp::I16x8ExtmulHighI8x16U): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI8x16)); + case uint32_t(SimdOp::I32x4ExtmulLowI16x8S): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowI16x8)); + case uint32_t(SimdOp::I32x4ExtmulHighI16x8S): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighI16x8)); + case uint32_t(SimdOp::I32x4ExtmulLowI16x8U): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI16x8)); + case uint32_t(SimdOp::I32x4ExtmulHighI16x8U): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI16x8)); + case uint32_t(SimdOp::I64x2ExtmulLowI32x4S): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowI32x4)); + case uint32_t(SimdOp::I64x2ExtmulHighI32x4S): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighI32x4)); + case uint32_t(SimdOp::I64x2ExtmulLowI32x4U): + CHECK_NEXT(dispatchVectorBinary(ExtMulLowUI32x4)); + case uint32_t(SimdOp::I64x2ExtmulHighI32x4U): + CHECK_NEXT(dispatchVectorBinary(ExtMulHighUI32x4)); + case uint32_t(SimdOp::I16x8Q15MulrSatS): + CHECK_NEXT(dispatchVectorBinary(Q15MulrSatS)); + case uint32_t(SimdOp::I8x16Neg): + CHECK_NEXT(dispatchVectorUnary(NegI8x16)); + case uint32_t(SimdOp::I16x8Neg): + CHECK_NEXT(dispatchVectorUnary(NegI16x8)); + case uint32_t(SimdOp::I16x8ExtendLowI8x16S): + CHECK_NEXT(dispatchVectorUnary(WidenLowI8x16)); + case uint32_t(SimdOp::I16x8ExtendHighI8x16S): + CHECK_NEXT(dispatchVectorUnary(WidenHighI8x16)); + case uint32_t(SimdOp::I16x8ExtendLowI8x16U): + CHECK_NEXT(dispatchVectorUnary(WidenLowUI8x16)); + case uint32_t(SimdOp::I16x8ExtendHighI8x16U): + CHECK_NEXT(dispatchVectorUnary(WidenHighUI8x16)); + case uint32_t(SimdOp::I32x4Neg): + CHECK_NEXT(dispatchVectorUnary(NegI32x4)); + case uint32_t(SimdOp::I32x4ExtendLowI16x8S): + CHECK_NEXT(dispatchVectorUnary(WidenLowI16x8)); + case uint32_t(SimdOp::I32x4ExtendHighI16x8S): + CHECK_NEXT(dispatchVectorUnary(WidenHighI16x8)); + case uint32_t(SimdOp::I32x4ExtendLowI16x8U): + CHECK_NEXT(dispatchVectorUnary(WidenLowUI16x8)); + case uint32_t(SimdOp::I32x4ExtendHighI16x8U): + CHECK_NEXT(dispatchVectorUnary(WidenHighUI16x8)); + case uint32_t(SimdOp::I32x4TruncSatF32x4S): + CHECK_NEXT(dispatchVectorUnary(ConvertF32x4ToI32x4)); + case uint32_t(SimdOp::I32x4TruncSatF32x4U): + CHECK_NEXT(dispatchVectorUnary(ConvertF32x4ToUI32x4)); + case uint32_t(SimdOp::I64x2Neg): + CHECK_NEXT(dispatchVectorUnary(NegI64x2)); + case uint32_t(SimdOp::I64x2ExtendLowI32x4S): + CHECK_NEXT(dispatchVectorUnary(WidenLowI32x4)); + case uint32_t(SimdOp::I64x2ExtendHighI32x4S): + CHECK_NEXT(dispatchVectorUnary(WidenHighI32x4)); + case uint32_t(SimdOp::I64x2ExtendLowI32x4U): + CHECK_NEXT(dispatchVectorUnary(WidenLowUI32x4)); + case uint32_t(SimdOp::I64x2ExtendHighI32x4U): + CHECK_NEXT(dispatchVectorUnary(WidenHighUI32x4)); + case uint32_t(SimdOp::F32x4Abs): + CHECK_NEXT(dispatchVectorUnary(AbsF32x4)); + case uint32_t(SimdOp::F32x4Neg): + CHECK_NEXT(dispatchVectorUnary(NegF32x4)); + case uint32_t(SimdOp::F32x4Sqrt): + CHECK_NEXT(dispatchVectorUnary(SqrtF32x4)); + case uint32_t(SimdOp::F32x4ConvertI32x4S): + CHECK_NEXT(dispatchVectorUnary(ConvertI32x4ToF32x4)); + case uint32_t(SimdOp::F32x4ConvertI32x4U): + CHECK_NEXT(dispatchVectorUnary(ConvertUI32x4ToF32x4)); + case uint32_t(SimdOp::F32x4DemoteF64x2Zero): + CHECK_NEXT(dispatchVectorUnary(DemoteF64x2ToF32x4)); + case uint32_t(SimdOp::F64x2PromoteLowF32x4): + CHECK_NEXT(dispatchVectorUnary(PromoteF32x4ToF64x2)); + case uint32_t(SimdOp::F64x2ConvertLowI32x4S): + CHECK_NEXT(dispatchVectorUnary(ConvertI32x4ToF64x2)); + case uint32_t(SimdOp::F64x2ConvertLowI32x4U): + CHECK_NEXT(dispatchVectorUnary(ConvertUI32x4ToF64x2)); + case uint32_t(SimdOp::I32x4TruncSatF64x2SZero): + CHECK_NEXT(dispatchVectorUnary(ConvertF64x2ToI32x4)); + case uint32_t(SimdOp::I32x4TruncSatF64x2UZero): + CHECK_NEXT(dispatchVectorUnary(ConvertF64x2ToUI32x4)); + case uint32_t(SimdOp::F64x2Abs): + CHECK_NEXT(dispatchVectorUnary(AbsF64x2)); + case uint32_t(SimdOp::F64x2Neg): + CHECK_NEXT(dispatchVectorUnary(NegF64x2)); + case uint32_t(SimdOp::F64x2Sqrt): + CHECK_NEXT(dispatchVectorUnary(SqrtF64x2)); + case uint32_t(SimdOp::V128Not): + CHECK_NEXT(dispatchVectorUnary(NotV128)); + case uint32_t(SimdOp::I8x16Popcnt): + CHECK_NEXT(dispatchVectorUnary(PopcntI8x16)); + case uint32_t(SimdOp::I8x16Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI8x16)); + case uint32_t(SimdOp::I16x8Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI16x8)); + case uint32_t(SimdOp::I32x4Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI32x4)); + case uint32_t(SimdOp::I64x2Abs): + CHECK_NEXT(dispatchVectorUnary(AbsI64x2)); + case uint32_t(SimdOp::F32x4Ceil): + CHECK_NEXT(dispatchVectorUnary(CeilF32x4)); + case uint32_t(SimdOp::F32x4Floor): + CHECK_NEXT(dispatchVectorUnary(FloorF32x4)); + case uint32_t(SimdOp::F32x4Trunc): + CHECK_NEXT(dispatchVectorUnary(TruncF32x4)); + case uint32_t(SimdOp::F32x4Nearest): + CHECK_NEXT(dispatchVectorUnary(NearestF32x4)); + case uint32_t(SimdOp::F64x2Ceil): + CHECK_NEXT(dispatchVectorUnary(CeilF64x2)); + case uint32_t(SimdOp::F64x2Floor): + CHECK_NEXT(dispatchVectorUnary(FloorF64x2)); + case uint32_t(SimdOp::F64x2Trunc): + CHECK_NEXT(dispatchVectorUnary(TruncF64x2)); + case uint32_t(SimdOp::F64x2Nearest): + CHECK_NEXT(dispatchVectorUnary(NearestF64x2)); + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseI8x16)); + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseUI8x16)); + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseI16x8)); + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U): + CHECK_NEXT(dispatchVectorUnary(ExtAddPairwiseUI16x8)); + case uint32_t(SimdOp::I8x16Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI8x16)); + case uint32_t(SimdOp::I8x16ShrS): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI8x16)); + case uint32_t(SimdOp::I8x16ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI8x16)); + case uint32_t(SimdOp::I16x8Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI16x8)); + case uint32_t(SimdOp::I16x8ShrS): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI16x8)); + case uint32_t(SimdOp::I16x8ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI16x8)); + case uint32_t(SimdOp::I32x4Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI32x4)); + case uint32_t(SimdOp::I32x4ShrS): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI32x4)); + case uint32_t(SimdOp::I32x4ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI32x4)); + case uint32_t(SimdOp::I64x2Shl): + CHECK_NEXT(dispatchVectorVariableShift(ShiftLeftI64x2)); + case uint32_t(SimdOp::I64x2ShrS): +# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) + CHECK_NEXT(emitVectorShiftRightI64x2()); +# else + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightI64x2)); +# endif + case uint32_t(SimdOp::I64x2ShrU): + CHECK_NEXT(dispatchVectorVariableShift(ShiftRightUI64x2)); + case uint32_t(SimdOp::V128Bitselect): + CHECK_NEXT(dispatchTernary1(BitselectV128, ValType::V128)); + case uint32_t(SimdOp::I8x16Shuffle): + CHECK_NEXT(emitVectorShuffle()); + case uint32_t(SimdOp::V128Const): { + V128 v128; + CHECK(iter_.readV128Const(&v128)); + if (!deadCode_) { + pushV128(v128); + } + NEXT(); + } + case uint32_t(SimdOp::V128Load): + CHECK_NEXT(emitLoad(ValType::V128, Scalar::Simd128)); + case uint32_t(SimdOp::V128Load8Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Uint8)); + case uint32_t(SimdOp::V128Load16Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Uint16)); + case uint32_t(SimdOp::V128Load32Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Uint32)); + case uint32_t(SimdOp::V128Load64Splat): + CHECK_NEXT(emitLoadSplat(Scalar::Int64)); + case uint32_t(SimdOp::V128Load8x8S): + CHECK_NEXT(emitLoadExtend(Scalar::Int8)); + case uint32_t(SimdOp::V128Load8x8U): + CHECK_NEXT(emitLoadExtend(Scalar::Uint8)); + case uint32_t(SimdOp::V128Load16x4S): + CHECK_NEXT(emitLoadExtend(Scalar::Int16)); + case uint32_t(SimdOp::V128Load16x4U): + CHECK_NEXT(emitLoadExtend(Scalar::Uint16)); + case uint32_t(SimdOp::V128Load32x2S): + CHECK_NEXT(emitLoadExtend(Scalar::Int32)); + case uint32_t(SimdOp::V128Load32x2U): + CHECK_NEXT(emitLoadExtend(Scalar::Uint32)); + case uint32_t(SimdOp::V128Load32Zero): + CHECK_NEXT(emitLoadZero(Scalar::Float32)); + case uint32_t(SimdOp::V128Load64Zero): + CHECK_NEXT(emitLoadZero(Scalar::Float64)); + case uint32_t(SimdOp::V128Store): + CHECK_NEXT(emitStore(ValType::V128, Scalar::Simd128)); + case uint32_t(SimdOp::V128Load8Lane): + CHECK_NEXT(emitLoadLane(1)); + case uint32_t(SimdOp::V128Load16Lane): + CHECK_NEXT(emitLoadLane(2)); + case uint32_t(SimdOp::V128Load32Lane): + CHECK_NEXT(emitLoadLane(4)); + case uint32_t(SimdOp::V128Load64Lane): + CHECK_NEXT(emitLoadLane(8)); + case uint32_t(SimdOp::V128Store8Lane): + CHECK_NEXT(emitStoreLane(1)); + case uint32_t(SimdOp::V128Store16Lane): + CHECK_NEXT(emitStoreLane(2)); + case uint32_t(SimdOp::V128Store32Lane): + CHECK_NEXT(emitStoreLane(4)); + case uint32_t(SimdOp::V128Store64Lane): + CHECK_NEXT(emitStoreLane(8)); +# ifdef ENABLE_WASM_RELAXED_SIMD + case uint32_t(SimdOp::F32x4RelaxedFma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFmaF32x4, ValType::V128)); + case uint32_t(SimdOp::F32x4RelaxedFnma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFnmaF32x4, ValType::V128)); + case uint32_t(SimdOp::F64x2RelaxedFma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFmaF64x2, ValType::V128)); + case uint32_t(SimdOp::F64x2RelaxedFnma): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary2(RelaxedFnmaF64x2, ValType::V128)); + break; + case uint32_t(SimdOp::I8x16RelaxedLaneSelect): + case uint32_t(SimdOp::I16x8RelaxedLaneSelect): + case uint32_t(SimdOp::I32x4RelaxedLaneSelect): + case uint32_t(SimdOp::I64x2RelaxedLaneSelect): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitVectorLaneSelect()); + case uint32_t(SimdOp::F32x4RelaxedMin): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMinF32x4)); + case uint32_t(SimdOp::F32x4RelaxedMax): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMaxF32x4)); + case uint32_t(SimdOp::F64x2RelaxedMin): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMinF64x2)); + case uint32_t(SimdOp::F64x2RelaxedMax): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedMaxF64x2)); + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF32x4ToI32x4)); + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF32x4ToUI32x4)); + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF64x2ToI32x4)); + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorUnary(RelaxedConvertF64x2ToUI32x4)); + case uint32_t(SimdOp::I8x16RelaxedSwizzle): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedSwizzle)); + case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(RelaxedQ15MulrS)); + case uint32_t(SimdOp::I16x8DotI8x16I7x16S): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchVectorBinary(DotI8x16I7x16S)); + case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS): + if (!moduleEnv_.v128RelaxedEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(dispatchTernary0(emitDotI8x16I7x16AddS, ValType::V128)); +# endif + default: + break; + } // switch (op.b1) + return iter_.unrecognizedOpcode(&op); + } +#endif // ENABLE_WASM_SIMD + + // "Miscellaneous" operations + case uint16_t(Op::MiscPrefix): { + switch (op.b1) { + case uint32_t(MiscOp::I32TruncSatF32S): + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF32ToI32, + ValType::F32, ValType::I32)); + case uint32_t(MiscOp::I32TruncSatF32U): + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF32ToI32, + ValType::F32, ValType::I32)); + case uint32_t(MiscOp::I32TruncSatF64S): + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF64ToI32, + ValType::F64, ValType::I32)); + case uint32_t(MiscOp::I32TruncSatF64U): + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF64ToI32, + ValType::F64, ValType::I32)); + case uint32_t(MiscOp::I64TruncSatF32S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToInt64, ValType::F32, + ValType::I64)); +#else + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF32ToI64, + ValType::F32, ValType::I64)); +#endif + case uint32_t(MiscOp::I64TruncSatF32U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToUint64, ValType::F32, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF32ToI64, + ValType::F32, ValType::I64)); +#endif + case uint32_t(MiscOp::I64TruncSatF64S): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToInt64, ValType::F64, + ValType::I64)); +#else + CHECK_NEXT( + dispatchConversionOOM(emitTruncateF64ToI64, + ValType::F64, ValType::I64)); +#endif + case uint32_t(MiscOp::I64TruncSatF64U): +#ifdef RABALDR_FLOAT_TO_I64_CALLOUT + CHECK_NEXT(dispatchCalloutConversionOOM( + emitConvertFloatingToInt64Callout, + SymbolicAddress::SaturatingTruncateDoubleToUint64, ValType::F64, + ValType::I64)); +#else + CHECK_NEXT(dispatchConversionOOM( + emitTruncateF64ToI64, + ValType::F64, ValType::I64)); +#endif + case uint32_t(MiscOp::MemoryCopy): + CHECK_NEXT(emitMemCopy()); + case uint32_t(MiscOp::DataDrop): + CHECK_NEXT(emitDataOrElemDrop(/*isData=*/true)); + case uint32_t(MiscOp::MemoryFill): + CHECK_NEXT(emitMemFill()); +#ifdef ENABLE_WASM_MEMORY_CONTROL + case uint32_t(MiscOp::MemoryDiscard): { + if (!moduleEnv_.memoryControlEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + CHECK_NEXT(emitMemDiscard()); + } +#endif + case uint32_t(MiscOp::MemoryInit): + CHECK_NEXT(emitMemInit()); + case uint32_t(MiscOp::TableCopy): + CHECK_NEXT(emitTableCopy()); + case uint32_t(MiscOp::ElemDrop): + CHECK_NEXT(emitDataOrElemDrop(/*isData=*/false)); + case uint32_t(MiscOp::TableInit): + CHECK_NEXT(emitTableInit()); + case uint32_t(MiscOp::TableFill): + CHECK_NEXT(emitTableFill()); + case uint32_t(MiscOp::TableGrow): + CHECK_NEXT(emitTableGrow()); + case uint32_t(MiscOp::TableSize): + CHECK_NEXT(emitTableSize()); + default: + break; + } // switch (op.b1) + return iter_.unrecognizedOpcode(&op); + } + + // Thread operations + case uint16_t(Op::ThreadPrefix): { + // Though thread ops can be used on nonshared memories, we make them + // unavailable if shared memory has been disabled in the prefs, for + // maximum predictability and safety and consistency with JS. + if (moduleEnv_.sharedMemoryEnabled() == Shareable::False) { + return iter_.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(ThreadOp::Wake): + CHECK_NEXT(emitWake()); + + case uint32_t(ThreadOp::I32Wait): + CHECK_NEXT(emitWait(ValType::I32, 4)); + case uint32_t(ThreadOp::I64Wait): + CHECK_NEXT(emitWait(ValType::I64, 8)); + case uint32_t(ThreadOp::Fence): + CHECK_NEXT(emitFence()); + + case uint32_t(ThreadOp::I32AtomicLoad): + CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicLoad): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicLoad8U): + CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicLoad16U): + CHECK_NEXT(emitAtomicLoad(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicLoad8U): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicLoad16U): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicLoad32U): + CHECK_NEXT(emitAtomicLoad(ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicStore): + CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicStore): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicStore8U): + CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicStore16U): + CHECK_NEXT(emitAtomicStore(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicStore8U): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicStore16U): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicStore32U): + CHECK_NEXT(emitAtomicStore(ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicAdd): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I32AtomicAdd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I32AtomicAdd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchAddOp)); + + case uint32_t(ThreadOp::I32AtomicSub): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I32AtomicSub8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I32AtomicSub16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchSubOp)); + + case uint32_t(ThreadOp::I32AtomicAnd): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I32AtomicAnd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I32AtomicAnd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchAndOp)); + + case uint32_t(ThreadOp::I32AtomicOr): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I32AtomicOr8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I32AtomicOr16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchOrOp)); + + case uint32_t(ThreadOp::I32AtomicXor): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Int32, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Int64, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I32AtomicXor8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint8, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I32AtomicXor16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I32, Scalar::Uint16, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor8U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint8, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor16U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint16, AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor32U): + CHECK_NEXT( + emitAtomicRMW(ValType::I64, Scalar::Uint32, AtomicFetchXorOp)); + + case uint32_t(ThreadOp::I32AtomicXchg): + CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicXchg): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicXchg8U): + CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicXchg16U): + CHECK_NEXT(emitAtomicXchg(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicXchg8U): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicXchg16U): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicXchg32U): + CHECK_NEXT(emitAtomicXchg(ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicCmpXchg): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicCmpXchg): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicCmpXchg8U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicCmpXchg16U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicCmpXchg8U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicCmpXchg16U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicCmpXchg32U): + CHECK_NEXT(emitAtomicCmpXchg(ValType::I64, Scalar::Uint32)); + + default: + return iter_.unrecognizedOpcode(&op); + } + break; + } + + // asm.js and other private operations + case uint16_t(Op::MozPrefix): { + if (op.b1 != uint32_t(MozOp::Intrinsic) || + !moduleEnv_.intrinsicsEnabled()) { + return iter_.unrecognizedOpcode(&op); + } + // private intrinsic operations + CHECK_NEXT(emitIntrinsic()); + } + + default: + return iter_.unrecognizedOpcode(&op); + } + +#undef CHECK +#undef NEXT +#undef CHECK_NEXT +#undef CHECK_POINTER_COUNT +#undef dispatchBinary0 +#undef dispatchBinary1 +#undef dispatchBinary2 +#undef dispatchBinary3 +#undef dispatchUnary0 +#undef dispatchUnary1 +#undef dispatchUnary2 +#undef dispatchComparison0 +#undef dispatchConversion0 +#undef dispatchConversion1 +#undef dispatchConversionOOM +#undef dispatchCalloutConversionOOM +#undef dispatchIntDivCallout +#undef dispatchVectorBinary +#undef dispatchVectorUnary +#undef dispatchVectorComparison +#undef dispatchExtractLane +#undef dispatchReplaceLane +#undef dispatchSplat +#undef dispatchVectorReduction + + MOZ_CRASH("unreachable"); + } + + MOZ_CRASH("unreachable"); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Various helpers. + +void BaseCompiler::assertResultRegistersAvailable(ResultType type) { +#ifdef DEBUG + for (ABIResultIter iter(type); !iter.done(); iter.next()) { + ABIResult result = iter.cur(); + if (!result.inRegister()) { + return; + } + switch (result.type().kind()) { + case ValType::I32: + MOZ_ASSERT(isAvailableI32(RegI32(result.gpr()))); + break; + case ValType::I64: + MOZ_ASSERT(isAvailableI64(RegI64(result.gpr64()))); + break; + case ValType::V128: +# ifdef ENABLE_WASM_SIMD + MOZ_ASSERT(isAvailableV128(RegV128(result.fpr()))); + break; +# else + MOZ_CRASH("No SIMD support"); +# endif + case ValType::F32: + MOZ_ASSERT(isAvailableF32(RegF32(result.fpr()))); + break; + case ValType::F64: + MOZ_ASSERT(isAvailableF64(RegF64(result.fpr()))); + break; + case ValType::Ref: + MOZ_ASSERT(isAvailableRef(RegRef(result.gpr()))); + break; + } + } +#endif +} + +void BaseCompiler::saveTempPtr(const RegPtr& r) { + MOZ_ASSERT(!ra.isAvailablePtr(r)); + fr.pushGPR(r); + ra.freePtr(r); + MOZ_ASSERT(ra.isAvailablePtr(r)); +} + +void BaseCompiler::restoreTempPtr(const RegPtr& r) { + MOZ_ASSERT(ra.isAvailablePtr(r)); + ra.needPtr(r); + fr.popGPR(r); + MOZ_ASSERT(!ra.isAvailablePtr(r)); +} + +#ifdef DEBUG +void BaseCompiler::performRegisterLeakCheck() { + BaseRegAlloc::LeakCheck check(ra); + for (auto& item : stk_) { + switch (item.kind_) { + case Stk::RegisterI32: + check.addKnownI32(item.i32reg()); + break; + case Stk::RegisterI64: + check.addKnownI64(item.i64reg()); + break; + case Stk::RegisterF32: + check.addKnownF32(item.f32reg()); + break; + case Stk::RegisterF64: + check.addKnownF64(item.f64reg()); + break; +# ifdef ENABLE_WASM_SIMD + case Stk::RegisterV128: + check.addKnownV128(item.v128reg()); + break; +# endif + case Stk::RegisterRef: + check.addKnownRef(item.refReg()); + break; + default: + break; + } + } +} + +void BaseCompiler::assertStackInvariants() const { + if (deadCode_) { + // Nonlocal control flow can pass values in stack locations in a way that + // isn't accounted for by the value stack. In dead code, which occurs + // after unconditional non-local control flow, there is no invariant to + // assert. + return; + } + size_t size = 0; + for (const Stk& v : stk_) { + switch (v.kind()) { + case Stk::MemRef: + size += BaseStackFrame::StackSizeOfPtr; + break; + case Stk::MemI32: + size += BaseStackFrame::StackSizeOfPtr; + break; + case Stk::MemI64: + size += BaseStackFrame::StackSizeOfInt64; + break; + case Stk::MemF64: + size += BaseStackFrame::StackSizeOfDouble; + break; + case Stk::MemF32: + size += BaseStackFrame::StackSizeOfFloat; + break; +# ifdef ENABLE_WASM_SIMD + case Stk::MemV128: + size += BaseStackFrame::StackSizeOfV128; + break; +# endif + default: + MOZ_ASSERT(!v.isMem()); + break; + } + } + MOZ_ASSERT(size == fr.dynamicHeight()); +} + +void BaseCompiler::showStack(const char* who) const { + fprintf(stderr, "Stack at %s {{\n", who); + size_t n = 0; + for (const Stk& elem : stk_) { + fprintf(stderr, " [%zu] ", n++); + elem.showStackElem(); + fprintf(stderr, "\n"); + } + fprintf(stderr, "}}\n"); +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// Main compilation logic. + +bool BaseCompiler::emitFunction() { + AutoCreatedBy acb(masm, "(wasm)BaseCompiler::emitFunction"); + + if (!beginFunction()) { + return false; + } + + if (!emitBody()) { + return false; + } + + if (!endFunction()) { + return false; + } + + return true; +} + +BaseCompiler::BaseCompiler(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + const FuncCompileInput& func, + const ValTypeVector& locals, + const RegisterOffsets& trapExitLayout, + size_t trapExitLayoutNumWords, Decoder& decoder, + StkVector& stkSource, TempAllocator* alloc, + MacroAssembler* masm, StackMaps* stackMaps) + : // Environment + moduleEnv_(moduleEnv), + compilerEnv_(compilerEnv), + func_(func), + locals_(locals), + previousBreakablePoint_(UINT32_MAX), + stkSource_(stkSource), + // Output-only data structures + alloc_(alloc->fallible()), + masm(*masm), + // Compilation state + decoder_(decoder), + iter_(moduleEnv, decoder), + fr(*masm), + stackMapGenerator_(stackMaps, trapExitLayout, trapExitLayoutNumWords, + *masm), + deadCode_(false), + bceSafe_(0), + latentOp_(LatentOp::None), + latentType_(ValType::I32), + latentIntCmp_(Assembler::Equal), + latentDoubleCmp_(Assembler::DoubleEqual) { + // Our caller, BaselineCompileFunctions, will lend us the vector contents to + // use for the eval stack. To get hold of those contents, we'll temporarily + // installing an empty one in its place. + MOZ_ASSERT(stk_.empty()); + stk_.swap(stkSource_); + + // Assuming that previously processed wasm functions are well formed, the + // eval stack should now be empty. But empty it anyway; any non-emptyness + // at this point will cause chaos. + stk_.clear(); +} + +BaseCompiler::~BaseCompiler() { + stk_.swap(stkSource_); + // We've returned the eval stack vector contents to our caller, + // BaselineCompileFunctions. We expect the vector we get in return to be + // empty since that's what we swapped for the stack vector in our + // constructor. + MOZ_ASSERT(stk_.empty()); +} + +bool BaseCompiler::init() { + // We may lift this restriction in the future. + MOZ_ASSERT_IF(usesMemory() && isMem64(), !moduleEnv_.hugeMemoryEnabled()); + // asm.js is not supported in baseline + MOZ_ASSERT(!moduleEnv_.isAsmJS()); + // Only asm.js modules have call site line numbers + MOZ_ASSERT(func_.callSiteLineNums.empty()); + + ra.init(this); + + if (!SigD_.append(ValType::F64)) { + return false; + } + if (!SigF_.append(ValType::F32)) { + return false; + } + + ArgTypeVector args(funcType()); + return fr.setupLocals(locals_, args, compilerEnv_.debugEnabled(), + &localInfo_); +} + +FuncOffsets BaseCompiler::finish() { + MOZ_ASSERT(iter_.done(), "all bytes must be consumed"); + MOZ_ASSERT(stk_.empty()); + MOZ_ASSERT(stackMapGenerator_.memRefsOnStk == 0); + + masm.flushBuffer(); + + return offsets_; +} + +} // namespace wasm +} // namespace js + +bool js::wasm::BaselinePlatformSupport() { +#if defined(JS_CODEGEN_ARM) + // Simplifying assumption: require SDIV and UDIV. + // + // I have no good data on ARM populations allowing me to say that + // X% of devices in the market implement SDIV and UDIV. However, + // they are definitely implemented on the Cortex-A7 and Cortex-A15 + // and on all ARMv8 systems. + if (!HasIDIV()) { + return false; + } +#endif +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ + defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64) + return true; +#else + return false; +#endif +} + +bool js::wasm::BaselineCompileFunctions(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + LifoAlloc& lifo, + const FuncCompileInputVector& inputs, + CompiledCode* code, + UniqueChars* error) { + MOZ_ASSERT(compilerEnv.tier() == Tier::Baseline); + MOZ_ASSERT(moduleEnv.kind == ModuleKind::Wasm); + + // The MacroAssembler will sometimes access the jitContext. + + TempAllocator alloc(&lifo); + JitContext jitContext; + MOZ_ASSERT(IsCompilingWasm()); + WasmMacroAssembler masm(alloc, moduleEnv); + + // Swap in already-allocated empty vectors to avoid malloc/free. + MOZ_ASSERT(code->empty()); + if (!code->swap(masm)) { + return false; + } + + // Create a description of the stack layout created by GenerateTrapExit(). + RegisterOffsets trapExitLayout; + size_t trapExitLayoutNumWords; + GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords); + + // The compiler's operand stack. We reuse it across all functions so as to + // avoid malloc/free. Presize it to 128 elements in the hope of avoiding + // reallocation later. + StkVector stk; + if (!stk.reserve(128)) { + return false; + } + + for (const FuncCompileInput& func : inputs) { + Decoder d(func.begin, func.end, func.lineOrBytecode, error); + + // Build the local types vector. + + ValTypeVector locals; + if (!locals.appendAll(moduleEnv.funcs[func.index].type->args())) { + return false; + } + if (!DecodeLocalEntries(d, *moduleEnv.types, moduleEnv.features, &locals)) { + return false; + } + + // One-pass baseline compilation. + + BaseCompiler f(moduleEnv, compilerEnv, func, locals, trapExitLayout, + trapExitLayoutNumWords, d, stk, &alloc, &masm, + &code->stackMaps); + if (!f.init()) { + return false; + } + if (!f.emitFunction()) { + return false; + } + if (!code->codeRanges.emplaceBack(func.index, func.lineOrBytecode, + f.finish())) { + return false; + } + } + + masm.finish(); + if (masm.oom()) { + return false; + } + + return code->swap(masm); +} diff --git a/js/src/wasm/WasmBaselineCompile.h b/js/src/wasm/WasmBaselineCompile.h new file mode 100644 index 0000000000..cd4f74e560 --- /dev/null +++ b/js/src/wasm/WasmBaselineCompile.h @@ -0,0 +1,103 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef asmjs_wasm_baseline_compile_h +#define asmjs_wasm_baseline_compile_h + +#include "jit/ABIArgGenerator.h" +#include "wasm/WasmGenerator.h" + +namespace js { +namespace wasm { + +// Return whether BaselineCompileFunction can generate code on the current +// device. Usually you do *not* want to call this, you want +// BaselineAvailable(). +[[nodiscard]] bool BaselinePlatformSupport(); + +// Generate adequate code quickly. +[[nodiscard]] bool BaselineCompileFunctions( + const ModuleEnvironment& moduleEnv, const CompilerEnvironment& compilerEnv, + LifoAlloc& lifo, const FuncCompileInputVector& inputs, CompiledCode* code, + UniqueChars* error); + +// BaseLocalIter iterates over a vector of types of locals and provides offsets +// from the Frame address for those locals, and associated data. +// +// The implementation of BaseLocalIter is the property of the BaseStackFrame. +// But it is also exposed for eg the debugger to use. +class BaseLocalIter { + private: + using ConstValTypeRange = mozilla::Range; + + const ValTypeVector& locals_; + const ArgTypeVector& args_; + jit::WasmABIArgIter argsIter_; + size_t index_; + int32_t frameSize_; + int32_t nextFrameSize_; + int32_t frameOffset_; + int32_t stackResultPointerOffset_; + jit::MIRType mirType_; + bool done_; + + void settle(); + int32_t pushLocal(size_t nbytes); + + public: + BaseLocalIter(const ValTypeVector& locals, const ArgTypeVector& args, + bool debugEnabled); + void operator++(int); + bool done() const { return done_; } + + jit::MIRType mirType() const { + MOZ_ASSERT(!done_); + return mirType_; + } + int32_t frameOffset() const { + MOZ_ASSERT(!done_); + MOZ_ASSERT(frameOffset_ != INT32_MAX); + return frameOffset_; + } + size_t index() const { + MOZ_ASSERT(!done_); + return index_; + } + // The size in bytes taken up by the previous `index_` locals, also including + // fixed allocations like the DebugFrame and "hidden" locals like a spilled + // stack results pointer. + int32_t frameSize() const { return frameSize_; } + + int32_t stackResultPointerOffset() const { + MOZ_ASSERT(args_.hasSyntheticStackResultPointerArg()); + MOZ_ASSERT(stackResultPointerOffset_ != INT32_MAX); + return stackResultPointerOffset_; + } + +#ifdef DEBUG + bool isArg() const { + MOZ_ASSERT(!done_); + return !argsIter_.done(); + } +#endif +}; + +} // namespace wasm +} // namespace js + +#endif // asmjs_wasm_baseline_compile_h diff --git a/js/src/wasm/WasmBinary.cpp b/js/src/wasm/WasmBinary.cpp new file mode 100644 index 0000000000..ccf50f9d2d --- /dev/null +++ b/js/src/wasm/WasmBinary.cpp @@ -0,0 +1,335 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmBinary.h" + +#include "js/Printf.h" +#include "wasm/WasmValidate.h" + +using namespace js; +using namespace js::wasm; + +// Decoder implementation. + +bool Decoder::failf(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + UniqueChars str(JS_vsmprintf(msg, ap)); + va_end(ap); + if (!str) { + return false; + } + + return fail(str.get()); +} + +void Decoder::warnf(const char* msg, ...) { + if (!warnings_) { + return; + } + + va_list ap; + va_start(ap, msg); + UniqueChars str(JS_vsmprintf(msg, ap)); + va_end(ap); + if (!str) { + return; + } + + (void)warnings_->append(std::move(str)); +} + +bool Decoder::fail(size_t errorOffset, const char* msg) { + MOZ_ASSERT(error_); + UniqueChars strWithOffset(JS_smprintf("at offset %zu: %s", errorOffset, msg)); + if (!strWithOffset) { + return false; + } + + *error_ = std::move(strWithOffset); + return false; +} + +bool Decoder::readSectionHeader(uint8_t* id, SectionRange* range) { + if (!readFixedU8(id)) { + return false; + } + + uint32_t size; + if (!readVarU32(&size)) { + return false; + } + + range->start = currentOffset(); + range->size = size; + return true; +} + +bool Decoder::startSection(SectionId id, ModuleEnvironment* env, + MaybeSectionRange* range, const char* sectionName) { + MOZ_ASSERT(!*range); + + // Record state at beginning of section to allow rewinding to this point + // if, after skipping through several custom sections, we don't find the + // section 'id'. + const uint8_t* const initialCur = cur_; + const size_t initialCustomSectionsLength = env->customSections.length(); + + // Maintain a pointer to the current section that gets updated as custom + // sections are skipped. + const uint8_t* currentSectionStart = cur_; + + // Only start a section with 'id', skipping any custom sections before it. + + uint8_t idValue; + if (!readFixedU8(&idValue)) { + goto rewind; + } + + while (idValue != uint8_t(id)) { + if (idValue != uint8_t(SectionId::Custom)) { + goto rewind; + } + + // Rewind to the beginning of the current section since this is what + // skipCustomSection() assumes. + cur_ = currentSectionStart; + if (!skipCustomSection(env)) { + return false; + } + + // Having successfully skipped a custom section, consider the next + // section. + currentSectionStart = cur_; + if (!readFixedU8(&idValue)) { + goto rewind; + } + } + + // Don't check the size since the range of bytes being decoded might not + // contain the section body. (This is currently the case when streaming: the + // code section header is decoded with the module environment bytes, the + // body of the code section is streamed in separately.) + + uint32_t size; + if (!readVarU32(&size)) { + goto fail; + } + + range->emplace(); + (*range)->start = currentOffset(); + (*range)->size = size; + return true; + +rewind: + cur_ = initialCur; + env->customSections.shrinkTo(initialCustomSectionsLength); + return true; + +fail: + return failf("failed to start %s section", sectionName); +} + +bool Decoder::finishSection(const SectionRange& range, + const char* sectionName) { + if (resilientMode_) { + return true; + } + if (range.size != currentOffset() - range.start) { + return failf("byte size mismatch in %s section", sectionName); + } + return true; +} + +bool Decoder::startCustomSection(const char* expected, size_t expectedLength, + ModuleEnvironment* env, + MaybeSectionRange* range) { + // Record state at beginning of section to allow rewinding to this point + // if, after skipping through several custom sections, we don't find the + // section 'id'. + const uint8_t* const initialCur = cur_; + const size_t initialCustomSectionsLength = env->customSections.length(); + + while (true) { + // Try to start a custom section. If we can't, rewind to the beginning + // since we may have skipped several custom sections already looking for + // 'expected'. + if (!startSection(SectionId::Custom, env, range, "custom")) { + return false; + } + if (!*range) { + goto rewind; + } + + if (bytesRemain() < (*range)->size) { + goto fail; + } + + CustomSectionEnv sec; + if (!readVarU32(&sec.nameLength) || sec.nameLength > bytesRemain()) { + goto fail; + } + + sec.nameOffset = currentOffset(); + sec.payloadOffset = sec.nameOffset + sec.nameLength; + + uint32_t payloadEnd = (*range)->start + (*range)->size; + if (sec.payloadOffset > payloadEnd) { + goto fail; + } + + sec.payloadLength = payloadEnd - sec.payloadOffset; + + // Now that we have a valid custom section, record its offsets in the + // metadata which can be queried by the user via Module.customSections. + // Note: after an entry is appended, it may be popped if this loop or + // the loop in startSection needs to rewind. + if (!env->customSections.append(sec)) { + return false; + } + + // If this is the expected custom section, we're done. + if (!expected || (expectedLength == sec.nameLength && + !memcmp(cur_, expected, sec.nameLength))) { + cur_ += sec.nameLength; + return true; + } + + // Otherwise, blindly skip the custom section and keep looking. + skipAndFinishCustomSection(**range); + range->reset(); + } + MOZ_CRASH("unreachable"); + +rewind: + cur_ = initialCur; + env->customSections.shrinkTo(initialCustomSectionsLength); + return true; + +fail: + return fail("failed to start custom section"); +} + +void Decoder::finishCustomSection(const char* name, const SectionRange& range) { + MOZ_ASSERT(cur_ >= beg_); + MOZ_ASSERT(cur_ <= end_); + + if (error_ && *error_) { + warnf("in the '%s' custom section: %s", name, error_->get()); + skipAndFinishCustomSection(range); + return; + } + + uint32_t actualSize = currentOffset() - range.start; + if (range.size != actualSize) { + if (actualSize < range.size) { + warnf("in the '%s' custom section: %" PRIu32 " unconsumed bytes", name, + uint32_t(range.size - actualSize)); + } else { + warnf("in the '%s' custom section: %" PRIu32 + " bytes consumed past the end", + name, uint32_t(actualSize - range.size)); + } + skipAndFinishCustomSection(range); + return; + } + + // Nothing to do! (c.f. skipAndFinishCustomSection()) +} + +void Decoder::skipAndFinishCustomSection(const SectionRange& range) { + MOZ_ASSERT(cur_ >= beg_); + MOZ_ASSERT(cur_ <= end_); + cur_ = (beg_ + (range.start - offsetInModule_)) + range.size; + MOZ_ASSERT(cur_ <= end_); + clearError(); +} + +bool Decoder::skipCustomSection(ModuleEnvironment* env) { + MaybeSectionRange range; + if (!startCustomSection(nullptr, 0, env, &range)) { + return false; + } + if (!range) { + return fail("expected custom section"); + } + + skipAndFinishCustomSection(*range); + return true; +} + +bool Decoder::startNameSubsection(NameType nameType, + Maybe* endOffset) { + MOZ_ASSERT(!*endOffset); + + const uint8_t* const initialPosition = cur_; + + uint8_t nameTypeValue; + if (!readFixedU8(&nameTypeValue)) { + goto rewind; + } + + if (nameTypeValue != uint8_t(nameType)) { + goto rewind; + } + + uint32_t payloadLength; + if (!readVarU32(&payloadLength) || payloadLength > bytesRemain()) { + return fail("bad name subsection payload length"); + } + + *endOffset = Some(currentOffset() + payloadLength); + return true; + +rewind: + cur_ = initialPosition; + return true; +} + +bool Decoder::finishNameSubsection(uint32_t endOffset) { + uint32_t actual = currentOffset(); + if (endOffset != actual) { + return failf("bad name subsection length (endOffset: %" PRIu32 + ", actual: %" PRIu32 ")", + endOffset, actual); + } + + return true; +} + +bool Decoder::skipNameSubsection() { + uint8_t nameTypeValue; + if (!readFixedU8(&nameTypeValue)) { + return fail("unable to read name subsection id"); + } + + switch (nameTypeValue) { + case uint8_t(NameType::Module): + case uint8_t(NameType::Function): + return fail("out of order name subsections"); + default: + break; + } + + uint32_t payloadLength; + if (!readVarU32(&payloadLength) || !readBytes(payloadLength)) { + return fail("bad name subsection payload length"); + } + + return true; +} diff --git a/js/src/wasm/WasmBinary.h b/js/src/wasm/WasmBinary.h new file mode 100644 index 0000000000..04015c55a4 --- /dev/null +++ b/js/src/wasm/WasmBinary.h @@ -0,0 +1,890 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_binary_h +#define wasm_binary_h + +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" + +#include + +#include "js/WasmFeatures.h" + +#include "wasm/WasmCompile.h" +#include "wasm/WasmCompileArgs.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValType.h" + +namespace js { +namespace wasm { + +using mozilla::DebugOnly; +using mozilla::Maybe; + +struct ModuleEnvironment; + +// The Opcode compactly and safely represents the primary opcode plus any +// extension, with convenient predicates and accessors. + +class Opcode { + uint32_t bits_; + + public: + MOZ_IMPLICIT Opcode(Op op) : bits_(uint32_t(op)) { + static_assert(size_t(Op::Limit) == 256, "fits"); + MOZ_ASSERT(size_t(op) < size_t(Op::Limit)); + } + MOZ_IMPLICIT Opcode(MiscOp op) + : bits_((uint32_t(op) << 8) | uint32_t(Op::MiscPrefix)) { + static_assert(size_t(MiscOp::Limit) <= 0xFFFFFF, "fits"); + MOZ_ASSERT(size_t(op) < size_t(MiscOp::Limit)); + } + MOZ_IMPLICIT Opcode(ThreadOp op) + : bits_((uint32_t(op) << 8) | uint32_t(Op::ThreadPrefix)) { + static_assert(size_t(ThreadOp::Limit) <= 0xFFFFFF, "fits"); + MOZ_ASSERT(size_t(op) < size_t(ThreadOp::Limit)); + } + MOZ_IMPLICIT Opcode(MozOp op) + : bits_((uint32_t(op) << 8) | uint32_t(Op::MozPrefix)) { + static_assert(size_t(MozOp::Limit) <= 0xFFFFFF, "fits"); + MOZ_ASSERT(size_t(op) < size_t(MozOp::Limit)); + } + MOZ_IMPLICIT Opcode(SimdOp op) + : bits_((uint32_t(op) << 8) | uint32_t(Op::SimdPrefix)) { + static_assert(size_t(SimdOp::Limit) <= 0xFFFFFF, "fits"); + MOZ_ASSERT(size_t(op) < size_t(SimdOp::Limit)); + } + + bool isOp() const { return bits_ < uint32_t(Op::FirstPrefix); } + bool isMisc() const { return (bits_ & 255) == uint32_t(Op::MiscPrefix); } + bool isThread() const { return (bits_ & 255) == uint32_t(Op::ThreadPrefix); } + bool isMoz() const { return (bits_ & 255) == uint32_t(Op::MozPrefix); } + bool isSimd() const { return (bits_ & 255) == uint32_t(Op::SimdPrefix); } + + Op asOp() const { + MOZ_ASSERT(isOp()); + return Op(bits_); + } + MiscOp asMisc() const { + MOZ_ASSERT(isMisc()); + return MiscOp(bits_ >> 8); + } + ThreadOp asThread() const { + MOZ_ASSERT(isThread()); + return ThreadOp(bits_ >> 8); + } + MozOp asMoz() const { + MOZ_ASSERT(isMoz()); + return MozOp(bits_ >> 8); + } + SimdOp asSimd() const { + MOZ_ASSERT(isSimd()); + return SimdOp(bits_ >> 8); + } + + uint32_t bits() const { return bits_; } + + bool operator==(const Opcode& that) const { return bits_ == that.bits_; } + bool operator!=(const Opcode& that) const { return bits_ != that.bits_; } +}; + +// This struct captures the bytecode offset of a section's payload (so not +// including the header) and the size of the payload. + +struct SectionRange { + uint32_t start; + uint32_t size; + + uint32_t end() const { return start + size; } + bool operator==(const SectionRange& rhs) const { + return start == rhs.start && size == rhs.size; + } +}; + +using MaybeSectionRange = Maybe; + +// The Encoder class appends bytes to the Bytes object it is given during +// construction. The client is responsible for the Bytes's lifetime and must +// keep the Bytes alive as long as the Encoder is used. + +class Encoder { + Bytes& bytes_; + + template + [[nodiscard]] bool write(const T& v) { + return bytes_.append(reinterpret_cast(&v), sizeof(T)); + } + + template + [[nodiscard]] bool writeVarU(UInt i) { + do { + uint8_t byte = i & 0x7f; + i >>= 7; + if (i != 0) { + byte |= 0x80; + } + if (!bytes_.append(byte)) { + return false; + } + } while (i != 0); + return true; + } + + template + [[nodiscard]] bool writeVarS(SInt i) { + bool done; + do { + uint8_t byte = i & 0x7f; + i >>= 7; + done = ((i == 0) && !(byte & 0x40)) || ((i == -1) && (byte & 0x40)); + if (!done) { + byte |= 0x80; + } + if (!bytes_.append(byte)) { + return false; + } + } while (!done); + return true; + } + + void patchVarU32(size_t offset, uint32_t patchBits, uint32_t assertBits) { + do { + uint8_t assertByte = assertBits & 0x7f; + uint8_t patchByte = patchBits & 0x7f; + assertBits >>= 7; + patchBits >>= 7; + if (assertBits != 0) { + assertByte |= 0x80; + patchByte |= 0x80; + } + MOZ_ASSERT(assertByte == bytes_[offset]); + (void)assertByte; + bytes_[offset] = patchByte; + offset++; + } while (assertBits != 0); + } + + void patchFixedU7(size_t offset, uint8_t patchBits, uint8_t assertBits) { + MOZ_ASSERT(patchBits <= uint8_t(INT8_MAX)); + patchFixedU8(offset, patchBits, assertBits); + } + + void patchFixedU8(size_t offset, uint8_t patchBits, uint8_t assertBits) { + MOZ_ASSERT(bytes_[offset] == assertBits); + bytes_[offset] = patchBits; + } + + uint32_t varU32ByteLength(size_t offset) const { + size_t start = offset; + while (bytes_[offset] & 0x80) { + offset++; + } + return offset - start + 1; + } + + public: + explicit Encoder(Bytes& bytes) : bytes_(bytes) { MOZ_ASSERT(empty()); } + + size_t currentOffset() const { return bytes_.length(); } + bool empty() const { return currentOffset() == 0; } + + // Fixed-size encoding operations simply copy the literal bytes (without + // attempting to align). + + [[nodiscard]] bool writeFixedU7(uint8_t i) { + MOZ_ASSERT(i <= uint8_t(INT8_MAX)); + return writeFixedU8(i); + } + [[nodiscard]] bool writeFixedU8(uint8_t i) { return write(i); } + [[nodiscard]] bool writeFixedU32(uint32_t i) { return write(i); } + [[nodiscard]] bool writeFixedF32(float f) { return write(f); } + [[nodiscard]] bool writeFixedF64(double d) { return write(d); } + + // Variable-length encodings that all use LEB128. + + [[nodiscard]] bool writeVarU32(uint32_t i) { return writeVarU(i); } + [[nodiscard]] bool writeVarS32(int32_t i) { return writeVarS(i); } + [[nodiscard]] bool writeVarU64(uint64_t i) { return writeVarU(i); } + [[nodiscard]] bool writeVarS64(int64_t i) { return writeVarS(i); } + [[nodiscard]] bool writeValType(ValType type) { + static_assert(size_t(TypeCode::Limit) <= UINT8_MAX, "fits"); + // writeValType is only used by asm.js, which doesn't use type + // references + MOZ_RELEASE_ASSERT(!type.isTypeRef(), "NYI"); + TypeCode tc = type.packed().typeCode(); + MOZ_ASSERT(size_t(tc) < size_t(TypeCode::Limit)); + return writeFixedU8(uint8_t(tc)); + } + [[nodiscard]] bool writeOp(Opcode opcode) { + // The Opcode constructor has asserted that `opcode` is meaningful, so no + // further correctness checking is necessary here. + uint32_t bits = opcode.bits(); + if (!writeFixedU8(bits & 255)) { + return false; + } + if (opcode.isOp()) { + return true; + } + return writeVarU32(bits >> 8); + } + + // Fixed-length encodings that allow back-patching. + + [[nodiscard]] bool writePatchableFixedU7(size_t* offset) { + *offset = bytes_.length(); + return writeFixedU8(UINT8_MAX); + } + void patchFixedU7(size_t offset, uint8_t patchBits) { + return patchFixedU7(offset, patchBits, UINT8_MAX); + } + + // Variable-length encodings that allow back-patching. + + [[nodiscard]] bool writePatchableVarU32(size_t* offset) { + *offset = bytes_.length(); + return writeVarU32(UINT32_MAX); + } + void patchVarU32(size_t offset, uint32_t patchBits) { + return patchVarU32(offset, patchBits, UINT32_MAX); + } + + // Byte ranges start with an LEB128 length followed by an arbitrary sequence + // of bytes. When used for strings, bytes are to be interpreted as utf8. + + [[nodiscard]] bool writeBytes(const void* bytes, uint32_t numBytes) { + return writeVarU32(numBytes) && + bytes_.append(reinterpret_cast(bytes), numBytes); + } + + // A "section" is a contiguous range of bytes that stores its own size so + // that it may be trivially skipped without examining the payload. Sections + // require backpatching since the size of the section is only known at the + // end while the size's varU32 must be stored at the beginning. Immediately + // after the section length is the string id of the section. + + [[nodiscard]] bool startSection(SectionId id, size_t* offset) { + MOZ_ASSERT(uint32_t(id) < 128); + return writeVarU32(uint32_t(id)) && writePatchableVarU32(offset); + } + void finishSection(size_t offset) { + return patchVarU32(offset, + bytes_.length() - offset - varU32ByteLength(offset)); + } +}; + +// The Decoder class decodes the bytes in the range it is given during +// construction. The client is responsible for keeping the byte range alive as +// long as the Decoder is used. + +class Decoder { + const uint8_t* const beg_; + const uint8_t* const end_; + const uint8_t* cur_; + const size_t offsetInModule_; + UniqueChars* error_; + UniqueCharsVector* warnings_; + bool resilientMode_; + + template + [[nodiscard]] bool read(T* out) { + if (bytesRemain() < sizeof(T)) { + return false; + } + memcpy((void*)out, cur_, sizeof(T)); + cur_ += sizeof(T); + return true; + } + + template + T uncheckedRead() { + MOZ_ASSERT(bytesRemain() >= sizeof(T)); + T ret; + memcpy(&ret, cur_, sizeof(T)); + cur_ += sizeof(T); + return ret; + } + + template + void uncheckedRead(T* ret) { + MOZ_ASSERT(bytesRemain() >= sizeof(T)); + memcpy(ret, cur_, sizeof(T)); + cur_ += sizeof(T); + } + + template + [[nodiscard]] bool readVarU(UInt* out) { + DebugOnly before = cur_; + const unsigned numBits = sizeof(UInt) * CHAR_BIT; + const unsigned remainderBits = numBits % 7; + const unsigned numBitsInSevens = numBits - remainderBits; + UInt u = 0; + uint8_t byte; + UInt shift = 0; + do { + if (!readFixedU8(&byte)) { + return false; + } + if (!(byte & 0x80)) { + *out = u | UInt(byte) << shift; + return true; + } + u |= UInt(byte & 0x7F) << shift; + shift += 7; + } while (shift != numBitsInSevens); + if (!readFixedU8(&byte) || (byte & (unsigned(-1) << remainderBits))) { + return false; + } + *out = u | (UInt(byte) << numBitsInSevens); + MOZ_ASSERT_IF(sizeof(UInt) == 4, + unsigned(cur_ - before) <= MaxVarU32DecodedBytes); + return true; + } + + template + [[nodiscard]] bool readVarS(SInt* out) { + using UInt = std::make_unsigned_t; + const unsigned numBits = sizeof(SInt) * CHAR_BIT; + const unsigned remainderBits = numBits % 7; + const unsigned numBitsInSevens = numBits - remainderBits; + SInt s = 0; + uint8_t byte; + unsigned shift = 0; + do { + if (!readFixedU8(&byte)) { + return false; + } + s |= SInt(byte & 0x7f) << shift; + shift += 7; + if (!(byte & 0x80)) { + if (byte & 0x40) { + s |= UInt(-1) << shift; + } + *out = s; + return true; + } + } while (shift < numBitsInSevens); + if (!remainderBits || !readFixedU8(&byte) || (byte & 0x80)) { + return false; + } + uint8_t mask = 0x7f & (uint8_t(-1) << remainderBits); + if ((byte & mask) != ((byte & (1 << (remainderBits - 1))) ? mask : 0)) { + return false; + } + *out = s | UInt(byte) << shift; + return true; + } + + public: + Decoder(const uint8_t* begin, const uint8_t* end, size_t offsetInModule, + UniqueChars* error, UniqueCharsVector* warnings = nullptr, + bool resilientMode = false) + : beg_(begin), + end_(end), + cur_(begin), + offsetInModule_(offsetInModule), + error_(error), + warnings_(warnings), + resilientMode_(resilientMode) { + MOZ_ASSERT(begin <= end); + } + explicit Decoder(const Bytes& bytes, size_t offsetInModule = 0, + UniqueChars* error = nullptr, + UniqueCharsVector* warnings = nullptr) + : beg_(bytes.begin()), + end_(bytes.end()), + cur_(bytes.begin()), + offsetInModule_(offsetInModule), + error_(error), + warnings_(warnings), + resilientMode_(false) {} + + // These convenience functions use currentOffset() as the errorOffset. + bool fail(const char* msg) { return fail(currentOffset(), msg); } + bool failf(const char* msg, ...) MOZ_FORMAT_PRINTF(2, 3); + void warnf(const char* msg, ...) MOZ_FORMAT_PRINTF(2, 3); + + // Report an error at the given offset (relative to the whole module). + bool fail(size_t errorOffset, const char* msg); + + UniqueChars* error() { return error_; } + + void clearError() { + if (error_) { + error_->reset(); + } + } + + bool done() const { + MOZ_ASSERT(cur_ <= end_); + return cur_ == end_; + } + bool resilientMode() const { return resilientMode_; } + + size_t bytesRemain() const { + MOZ_ASSERT(end_ >= cur_); + return size_t(end_ - cur_); + } + // pos must be a value previously returned from currentPosition. + void rollbackPosition(const uint8_t* pos) { cur_ = pos; } + const uint8_t* currentPosition() const { return cur_; } + size_t beginOffset() const { return offsetInModule_; } + size_t currentOffset() const { return offsetInModule_ + (cur_ - beg_); } + const uint8_t* begin() const { return beg_; } + const uint8_t* end() const { return end_; } + + // Peek at the next byte, if it exists, without advancing the position. + + bool peekByte(uint8_t* byte) { + if (done()) { + return false; + } + *byte = *cur_; + return true; + } + + // Fixed-size encoding operations simply copy the literal bytes (without + // attempting to align). + + [[nodiscard]] bool readFixedU8(uint8_t* i) { return read(i); } + [[nodiscard]] bool readFixedU32(uint32_t* u) { return read(u); } + [[nodiscard]] bool readFixedF32(float* f) { return read(f); } + [[nodiscard]] bool readFixedF64(double* d) { return read(d); } +#ifdef ENABLE_WASM_SIMD + [[nodiscard]] bool readFixedV128(V128* d) { + for (unsigned i = 0; i < 16; i++) { + if (!read(d->bytes + i)) { + return false; + } + } + return true; + } +#endif + + // Variable-length encodings that all use LEB128. + + [[nodiscard]] bool readVarU32(uint32_t* out) { + return readVarU(out); + } + [[nodiscard]] bool readVarS32(int32_t* out) { return readVarS(out); } + [[nodiscard]] bool readVarU64(uint64_t* out) { + return readVarU(out); + } + [[nodiscard]] bool readVarS64(int64_t* out) { return readVarS(out); } + + // Value and reference types + + [[nodiscard]] ValType uncheckedReadValType(const TypeContext& types); + + template + [[nodiscard]] bool readPackedType(const TypeContext& types, + const FeatureArgs& features, T* type); + + [[nodiscard]] bool readValType(const TypeContext& types, + const FeatureArgs& features, ValType* type); + + [[nodiscard]] bool readFieldType(const TypeContext& types, + const FeatureArgs& features, + FieldType* type); + + [[nodiscard]] bool readHeapType(const TypeContext& types, + const FeatureArgs& features, bool nullable, + RefType* type); + + [[nodiscard]] bool readRefType(const TypeContext& types, + const FeatureArgs& features, RefType* type); + + // Instruction opcode + + [[nodiscard]] bool readOp(OpBytes* op); + + // Instruction immediates for constant instructions + + [[nodiscard]] bool readBinary() { return true; } + [[nodiscard]] bool readTypeIndex(uint32_t* typeIndex); + [[nodiscard]] bool readGlobalIndex(uint32_t* globalIndex); + [[nodiscard]] bool readFuncIndex(uint32_t* funcIndex); + [[nodiscard]] bool readI32Const(int32_t* i32); + [[nodiscard]] bool readI64Const(int64_t* i64); + [[nodiscard]] bool readF32Const(float* f32); + [[nodiscard]] bool readF64Const(double* f64); +#ifdef ENABLE_WASM_SIMD + [[nodiscard]] bool readV128Const(V128* value); +#endif + [[nodiscard]] bool readRefNull(const TypeContext& types, + const FeatureArgs& features, RefType* type); + + // See writeBytes comment. + + [[nodiscard]] bool readBytes(uint32_t numBytes, + const uint8_t** bytes = nullptr) { + if (bytes) { + *bytes = cur_; + } + if (bytesRemain() < numBytes) { + return false; + } + cur_ += numBytes; + return true; + } + + // See "section" description in Encoder. + + [[nodiscard]] bool readSectionHeader(uint8_t* id, SectionRange* range); + + [[nodiscard]] bool startSection(SectionId id, ModuleEnvironment* env, + MaybeSectionRange* range, + const char* sectionName); + [[nodiscard]] bool finishSection(const SectionRange& range, + const char* sectionName); + + // Custom sections do not cause validation errors unless the error is in + // the section header itself. + + [[nodiscard]] bool startCustomSection(const char* expected, + size_t expectedLength, + ModuleEnvironment* env, + MaybeSectionRange* range); + + template + [[nodiscard]] bool startCustomSection(const char (&name)[NameSizeWith0], + ModuleEnvironment* env, + MaybeSectionRange* range) { + MOZ_ASSERT(name[NameSizeWith0 - 1] == '\0'); + return startCustomSection(name, NameSizeWith0 - 1, env, range); + } + + void finishCustomSection(const char* name, const SectionRange& range); + void skipAndFinishCustomSection(const SectionRange& range); + + [[nodiscard]] bool skipCustomSection(ModuleEnvironment* env); + + // The Name section has its own optional subsections. + + [[nodiscard]] bool startNameSubsection(NameType nameType, + Maybe* endOffset); + [[nodiscard]] bool finishNameSubsection(uint32_t endOffset); + [[nodiscard]] bool skipNameSubsection(); + + // The infallible "unchecked" decoding functions can be used when we are + // sure that the bytes are well-formed (by construction or due to previous + // validation). + + uint8_t uncheckedReadFixedU8() { return uncheckedRead(); } + uint32_t uncheckedReadFixedU32() { return uncheckedRead(); } + void uncheckedReadFixedF32(float* out) { uncheckedRead(out); } + void uncheckedReadFixedF64(double* out) { uncheckedRead(out); } + template + UInt uncheckedReadVarU() { + static const unsigned numBits = sizeof(UInt) * CHAR_BIT; + static const unsigned remainderBits = numBits % 7; + static const unsigned numBitsInSevens = numBits - remainderBits; + UInt decoded = 0; + uint32_t shift = 0; + do { + uint8_t byte = *cur_++; + if (!(byte & 0x80)) { + return decoded | (UInt(byte) << shift); + } + decoded |= UInt(byte & 0x7f) << shift; + shift += 7; + } while (shift != numBitsInSevens); + uint8_t byte = *cur_++; + MOZ_ASSERT(!(byte & 0xf0)); + return decoded | (UInt(byte) << numBitsInSevens); + } + uint32_t uncheckedReadVarU32() { return uncheckedReadVarU(); } + int32_t uncheckedReadVarS32() { + int32_t i32 = 0; + MOZ_ALWAYS_TRUE(readVarS32(&i32)); + return i32; + } + uint64_t uncheckedReadVarU64() { return uncheckedReadVarU(); } + int64_t uncheckedReadVarS64() { + int64_t i64 = 0; + MOZ_ALWAYS_TRUE(readVarS64(&i64)); + return i64; + } + Op uncheckedReadOp() { + static_assert(size_t(Op::Limit) == 256, "fits"); + uint8_t u8 = uncheckedReadFixedU8(); + return u8 != UINT8_MAX ? Op(u8) : Op(uncheckedReadFixedU8() + UINT8_MAX); + } +}; + +// Value and reference types + +inline ValType Decoder::uncheckedReadValType(const TypeContext& types) { + uint8_t code = uncheckedReadFixedU8(); + switch (code) { + case uint8_t(TypeCode::FuncRef): + case uint8_t(TypeCode::ExternRef): + return RefType::fromTypeCode(TypeCode(code), true); + case uint8_t(TypeCode::Ref): + case uint8_t(TypeCode::NullableRef): { + bool nullable = code == uint8_t(TypeCode::NullableRef); + + uint8_t nextByte; + peekByte(&nextByte); + + if ((nextByte & SLEB128SignMask) == SLEB128SignBit) { + uint8_t code = uncheckedReadFixedU8(); + return RefType::fromTypeCode(TypeCode(code), nullable); + } + + int32_t x = uncheckedReadVarS32(); + const TypeDef* typeDef = &types.type(x); + return RefType::fromTypeDef(typeDef, nullable); + } + default: + return ValType::fromNonRefTypeCode(TypeCode(code)); + } +} + +template +inline bool Decoder::readPackedType(const TypeContext& types, + const FeatureArgs& features, T* type) { + static_assert(uint8_t(TypeCode::Limit) <= UINT8_MAX, "fits"); + uint8_t code; + if (!readFixedU8(&code)) { + return fail("expected type code"); + } + switch (code) { + case uint8_t(TypeCode::V128): { +#ifdef ENABLE_WASM_SIMD + if (!features.simd) { + return fail("v128 not enabled"); + } + *type = T::fromNonRefTypeCode(TypeCode(code)); + return true; +#else + break; +#endif + } + case uint8_t(TypeCode::FuncRef): + case uint8_t(TypeCode::ExternRef): { + *type = RefType::fromTypeCode(TypeCode(code), true); + return true; + } + case uint8_t(TypeCode::Ref): + case uint8_t(TypeCode::NullableRef): { +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + if (!features.functionReferences) { + return fail("(ref T) types not enabled"); + } + bool nullable = code == uint8_t(TypeCode::NullableRef); + RefType refType; + if (!readHeapType(types, features, nullable, &refType)) { + return false; + } + *type = refType; + return true; +#else + break; +#endif + } + case uint8_t(TypeCode::AnyRef): + case uint8_t(TypeCode::EqRef): + case uint8_t(TypeCode::StructRef): + case uint8_t(TypeCode::ArrayRef): + case uint8_t(TypeCode::NullFuncRef): + case uint8_t(TypeCode::NullExternRef): + case uint8_t(TypeCode::NullAnyRef): { +#ifdef ENABLE_WASM_GC + if (!features.gc) { + return fail("gc types not enabled"); + } + *type = RefType::fromTypeCode(TypeCode(code), true); + return true; +#else + break; +#endif + } + default: { + if (!T::isValidTypeCode(TypeCode(code))) { + break; + } + *type = T::fromNonRefTypeCode(TypeCode(code)); + return true; + } + } + return fail("bad type"); +} + +inline bool Decoder::readValType(const TypeContext& types, + const FeatureArgs& features, ValType* type) { + return readPackedType(types, features, type); +} + +inline bool Decoder::readFieldType(const TypeContext& types, + const FeatureArgs& features, + FieldType* type) { + return readPackedType(types, features, type); +} + +inline bool Decoder::readHeapType(const TypeContext& types, + const FeatureArgs& features, bool nullable, + RefType* type) { + uint8_t nextByte; + if (!peekByte(&nextByte)) { + return fail("expected heap type code"); + } + + if ((nextByte & SLEB128SignMask) == SLEB128SignBit) { + uint8_t code; + if (!readFixedU8(&code)) { + return false; + } + + switch (code) { + case uint8_t(TypeCode::FuncRef): + case uint8_t(TypeCode::ExternRef): + *type = RefType::fromTypeCode(TypeCode(code), nullable); + return true; +#ifdef ENABLE_WASM_GC + case uint8_t(TypeCode::AnyRef): + case uint8_t(TypeCode::EqRef): + case uint8_t(TypeCode::StructRef): + case uint8_t(TypeCode::ArrayRef): + case uint8_t(TypeCode::NullFuncRef): + case uint8_t(TypeCode::NullExternRef): + case uint8_t(TypeCode::NullAnyRef): + if (!features.gc) { + return fail("gc types not enabled"); + } + *type = RefType::fromTypeCode(TypeCode(code), nullable); + return true; +#endif + default: + return fail("invalid heap type"); + } + } + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + if (features.functionReferences) { + int32_t x; + if (!readVarS32(&x) || x < 0 || uint32_t(x) >= types.length()) { + return fail("invalid heap type index"); + } + const TypeDef* typeDef = &types.type(x); + *type = RefType::fromTypeDef(typeDef, nullable); + return true; + } +#endif + return fail("invalid heap type"); +} + +inline bool Decoder::readRefType(const TypeContext& types, + const FeatureArgs& features, RefType* type) { + ValType valType; + if (!readValType(types, features, &valType)) { + return false; + } + if (!valType.isRefType()) { + return fail("bad type"); + } + *type = valType.refType(); + return true; +} + +// Instruction opcode + +inline bool Decoder::readOp(OpBytes* op) { + static_assert(size_t(Op::Limit) == 256, "fits"); + uint8_t u8; + if (!readFixedU8(&u8)) { + return false; + } + op->b0 = u8; + if (MOZ_LIKELY(!IsPrefixByte(u8))) { + return true; + } + return readVarU32(&op->b1); +} + +// Instruction immediates for constant instructions + +inline bool Decoder::readTypeIndex(uint32_t* typeIndex) { + if (!readVarU32(typeIndex)) { + return fail("unable to read type index"); + } + return true; +} + +inline bool Decoder::readGlobalIndex(uint32_t* globalIndex) { + if (!readVarU32(globalIndex)) { + return fail("unable to read global index"); + } + return true; +} + +inline bool Decoder::readFuncIndex(uint32_t* funcIndex) { + if (!readVarU32(funcIndex)) { + return fail("unable to read function index"); + } + return true; +} + +inline bool Decoder::readI32Const(int32_t* i32) { + if (!readVarS32(i32)) { + return fail("failed to read I32 constant"); + } + return true; +} + +inline bool Decoder::readI64Const(int64_t* i64) { + if (!readVarS64(i64)) { + return fail("failed to read I64 constant"); + } + return true; +} + +inline bool Decoder::readF32Const(float* f32) { + if (!readFixedF32(f32)) { + return fail("failed to read F32 constant"); + } + return true; +} + +inline bool Decoder::readF64Const(double* f64) { + if (!readFixedF64(f64)) { + return fail("failed to read F64 constant"); + } + return true; +} + +#ifdef ENABLE_WASM_SIMD +inline bool Decoder::readV128Const(V128* value) { + if (!readFixedV128(value)) { + return fail("unable to read V128 constant"); + } + return true; +} +#endif + +inline bool Decoder::readRefNull(const TypeContext& types, + const FeatureArgs& features, RefType* type) { + return readHeapType(types, features, true, type); +} + +} // namespace wasm +} // namespace js + +#endif // namespace wasm_binary_h diff --git a/js/src/wasm/WasmBuiltins.cpp b/js/src/wasm/WasmBuiltins.cpp new file mode 100644 index 0000000000..86cc45a9ed --- /dev/null +++ b/js/src/wasm/WasmBuiltins.cpp @@ -0,0 +1,1980 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2017 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmBuiltins.h" + +#include "mozilla/Atomics.h" + +#include "fdlibm.h" +#include "jslibmath.h" +#include "jsmath.h" + +#include "gc/Allocator.h" +#include "jit/AtomicOperations.h" +#include "jit/InlinableNatives.h" +#include "jit/MacroAssembler.h" +#include "jit/Simulator.h" +#include "js/experimental/JitInfo.h" // JSJitInfo +#include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_* +#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit +#include "threading/Mutex.h" +#include "util/Memory.h" +#include "util/Poison.h" +#include "vm/BigIntType.h" +#include "vm/ErrorObject.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmDebug.h" +#include "wasm/WasmDebugFrame.h" +#include "wasm/WasmGcObject.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmStubs.h" + +#include "debugger/DebugAPI-inl.h" +#include "vm/ErrorObject-inl.h" +#include "vm/Stack-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace jit; +using namespace wasm; + +using mozilla::HashGeneric; +using mozilla::MakeEnumeratedRange; + +static const unsigned BUILTIN_THUNK_LIFO_SIZE = 64 * 1024; + +// ============================================================================ +// WebAssembly builtin C++ functions called from wasm code to implement internal +// wasm operations: type descriptions. + +// Some abbreviations, for the sake of conciseness. +#define _F64 MIRType::Double +#define _F32 MIRType::Float32 +#define _I32 MIRType::Int32 +#define _I64 MIRType::Int64 +#define _PTR MIRType::Pointer +#define _RoN MIRType::RefOrNull +#define _VOID MIRType::None +#define _END MIRType::None +#define _Infallible FailureMode::Infallible +#define _FailOnNegI32 FailureMode::FailOnNegI32 +#define _FailOnNullPtr FailureMode::FailOnNullPtr +#define _FailOnInvalidRef FailureMode::FailOnInvalidRef + +namespace js { +namespace wasm { + +const SymbolicAddressSignature SASigSinNativeD = { + SymbolicAddress::SinNativeD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigSinFdlibmD = { + SymbolicAddress::SinFdlibmD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigCosNativeD = { + SymbolicAddress::CosNativeD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigCosFdlibmD = { + SymbolicAddress::CosFdlibmD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigTanNativeD = { + SymbolicAddress::TanNativeD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigTanFdlibmD = { + SymbolicAddress::TanFdlibmD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigASinD = { + SymbolicAddress::ASinD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigACosD = { + SymbolicAddress::ACosD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigATanD = { + SymbolicAddress::ATanD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigCeilD = { + SymbolicAddress::CeilD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigCeilF = { + SymbolicAddress::CeilF, _F32, _Infallible, 1, {_F32, _END}}; +const SymbolicAddressSignature SASigFloorD = { + SymbolicAddress::FloorD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigFloorF = { + SymbolicAddress::FloorF, _F32, _Infallible, 1, {_F32, _END}}; +const SymbolicAddressSignature SASigTruncD = { + SymbolicAddress::TruncD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigTruncF = { + SymbolicAddress::TruncF, _F32, _Infallible, 1, {_F32, _END}}; +const SymbolicAddressSignature SASigNearbyIntD = { + SymbolicAddress::NearbyIntD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigNearbyIntF = { + SymbolicAddress::NearbyIntF, _F32, _Infallible, 1, {_F32, _END}}; +const SymbolicAddressSignature SASigExpD = { + SymbolicAddress::ExpD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigLogD = { + SymbolicAddress::LogD, _F64, _Infallible, 1, {_F64, _END}}; +const SymbolicAddressSignature SASigPowD = { + SymbolicAddress::PowD, _F64, _Infallible, 2, {_F64, _F64, _END}}; +const SymbolicAddressSignature SASigATan2D = { + SymbolicAddress::ATan2D, _F64, _Infallible, 2, {_F64, _F64, _END}}; +const SymbolicAddressSignature SASigMemoryGrowM32 = { + SymbolicAddress::MemoryGrowM32, _I32, _Infallible, 2, {_PTR, _I32, _END}}; +const SymbolicAddressSignature SASigMemoryGrowM64 = { + SymbolicAddress::MemoryGrowM64, _I64, _Infallible, 2, {_PTR, _I64, _END}}; +const SymbolicAddressSignature SASigMemorySizeM32 = { + SymbolicAddress::MemorySizeM32, _I32, _Infallible, 1, {_PTR, _END}}; +const SymbolicAddressSignature SASigMemorySizeM64 = { + SymbolicAddress::MemorySizeM64, _I64, _Infallible, 1, {_PTR, _END}}; +const SymbolicAddressSignature SASigWaitI32M32 = { + SymbolicAddress::WaitI32M32, + _I32, + _FailOnNegI32, + 4, + {_PTR, _I32, _I32, _I64, _END}}; +const SymbolicAddressSignature SASigWaitI32M64 = { + SymbolicAddress::WaitI32M64, + _I32, + _FailOnNegI32, + 4, + {_PTR, _I64, _I32, _I64, _END}}; +const SymbolicAddressSignature SASigWaitI64M32 = { + SymbolicAddress::WaitI64M32, + _I32, + _FailOnNegI32, + 4, + {_PTR, _I32, _I64, _I64, _END}}; +const SymbolicAddressSignature SASigWaitI64M64 = { + SymbolicAddress::WaitI64M64, + _I32, + _FailOnNegI32, + 4, + {_PTR, _I64, _I64, _I64, _END}}; +const SymbolicAddressSignature SASigWakeM32 = { + SymbolicAddress::WakeM32, _I32, _FailOnNegI32, 3, {_PTR, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigWakeM64 = { + SymbolicAddress::WakeM64, _I32, _FailOnNegI32, 3, {_PTR, _I64, _I32, _END}}; +const SymbolicAddressSignature SASigMemCopyM32 = { + SymbolicAddress::MemCopyM32, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I32, _I32, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigMemCopySharedM32 = { + SymbolicAddress::MemCopySharedM32, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I32, _I32, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigMemCopyM64 = { + SymbolicAddress::MemCopyM64, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I64, _I64, _I64, _PTR, _END}}; +const SymbolicAddressSignature SASigMemCopySharedM64 = { + SymbolicAddress::MemCopySharedM64, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I64, _I64, _I64, _PTR, _END}}; +const SymbolicAddressSignature SASigDataDrop = { + SymbolicAddress::DataDrop, _VOID, _FailOnNegI32, 2, {_PTR, _I32, _END}}; +const SymbolicAddressSignature SASigMemFillM32 = { + SymbolicAddress::MemFillM32, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I32, _I32, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigMemFillSharedM32 = { + SymbolicAddress::MemFillSharedM32, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I32, _I32, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigMemFillM64 = { + SymbolicAddress::MemFillM64, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I64, _I32, _I64, _PTR, _END}}; +const SymbolicAddressSignature SASigMemFillSharedM64 = { + SymbolicAddress::MemFillSharedM64, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I64, _I32, _I64, _PTR, _END}}; +const SymbolicAddressSignature SASigMemDiscardM32 = { + SymbolicAddress::MemDiscardM32, + _VOID, + _FailOnNegI32, + 4, + {_PTR, _I32, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigMemDiscardSharedM32 = { + SymbolicAddress::MemDiscardSharedM32, + _VOID, + _FailOnNegI32, + 4, + {_PTR, _I32, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigMemDiscardM64 = { + SymbolicAddress::MemDiscardM64, + _VOID, + _FailOnNegI32, + 4, + {_PTR, _I64, _I64, _PTR, _END}}; +const SymbolicAddressSignature SASigMemDiscardSharedM64 = { + SymbolicAddress::MemDiscardSharedM64, + _VOID, + _FailOnNegI32, + 4, + {_PTR, _I64, _I64, _PTR, _END}}; +const SymbolicAddressSignature SASigMemInitM32 = { + SymbolicAddress::MemInitM32, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I32, _I32, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigMemInitM64 = { + SymbolicAddress::MemInitM64, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I64, _I32, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigTableCopy = { + SymbolicAddress::TableCopy, + _VOID, + _FailOnNegI32, + 6, + {_PTR, _I32, _I32, _I32, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigElemDrop = { + SymbolicAddress::ElemDrop, _VOID, _FailOnNegI32, 2, {_PTR, _I32, _END}}; +const SymbolicAddressSignature SASigTableFill = { + SymbolicAddress::TableFill, + _VOID, + _FailOnNegI32, + 5, + {_PTR, _I32, _RoN, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigTableGet = {SymbolicAddress::TableGet, + _RoN, + _FailOnInvalidRef, + 3, + {_PTR, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigTableGrow = { + SymbolicAddress::TableGrow, + _I32, + _Infallible, + 4, + {_PTR, _RoN, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigTableInit = { + SymbolicAddress::TableInit, + _VOID, + _FailOnNegI32, + 6, + {_PTR, _I32, _I32, _I32, _I32, _I32, _END}}; +const SymbolicAddressSignature SASigTableSet = {SymbolicAddress::TableSet, + _VOID, + _FailOnNegI32, + 4, + {_PTR, _I32, _RoN, _I32, _END}}; +const SymbolicAddressSignature SASigTableSize = { + SymbolicAddress::TableSize, _I32, _Infallible, 2, {_PTR, _I32, _END}}; +const SymbolicAddressSignature SASigRefFunc = { + SymbolicAddress::RefFunc, _RoN, _FailOnInvalidRef, 2, {_PTR, _I32, _END}}; +const SymbolicAddressSignature SASigPostBarrier = { + SymbolicAddress::PostBarrier, _VOID, _Infallible, 2, {_PTR, _PTR, _END}}; +const SymbolicAddressSignature SASigPostBarrierPrecise = { + SymbolicAddress::PostBarrierPrecise, + _VOID, + _Infallible, + 3, + {_PTR, _PTR, _RoN, _END}}; +const SymbolicAddressSignature SASigPostBarrierPreciseWithOffset = { + SymbolicAddress::PostBarrierPreciseWithOffset, + _VOID, + _Infallible, + 4, + {_PTR, _PTR, _I32, _RoN, _END}}; +const SymbolicAddressSignature SASigExceptionNew = { + SymbolicAddress::ExceptionNew, _RoN, _FailOnNullPtr, 2, {_PTR, _RoN, _END}}; +const SymbolicAddressSignature SASigThrowException = { + SymbolicAddress::ThrowException, + _VOID, + _FailOnNegI32, + 2, + {_PTR, _RoN, _END}}; +const SymbolicAddressSignature SASigStructNew = { + SymbolicAddress::StructNew, _RoN, _FailOnNullPtr, 2, {_PTR, _PTR, _END}}; +const SymbolicAddressSignature SASigStructNewUninit = { + SymbolicAddress::StructNewUninit, + _RoN, + _FailOnNullPtr, + 2, + {_PTR, _PTR, _END}}; +const SymbolicAddressSignature SASigArrayNew = {SymbolicAddress::ArrayNew, + _RoN, + _FailOnNullPtr, + 3, + {_PTR, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigArrayNewUninit = { + SymbolicAddress::ArrayNewUninit, + _RoN, + _FailOnNullPtr, + 3, + {_PTR, _I32, _PTR, _END}}; +const SymbolicAddressSignature SASigArrayNewData = { + SymbolicAddress::ArrayNewData, + _RoN, + _FailOnNullPtr, + 5, + {_PTR, _I32, _I32, _PTR, _I32, _END}}; +const SymbolicAddressSignature SASigArrayNewElem = { + SymbolicAddress::ArrayNewElem, + _RoN, + _FailOnNullPtr, + 5, + {_PTR, _I32, _I32, _PTR, _I32, _END}}; +const SymbolicAddressSignature SASigArrayCopy = { + SymbolicAddress::ArrayCopy, + _VOID, + _FailOnNegI32, + 7, + {_PTR, _RoN, _I32, _RoN, _I32, _I32, _I32, _END}}; + +#define DECL_SAS_FOR_INTRINSIC(op, export, sa_name, abitype, entry, idx) \ + const SymbolicAddressSignature SASig##sa_name = { \ + SymbolicAddress::sa_name, _VOID, _FailOnNegI32, \ + DECLARE_INTRINSIC_PARAM_TYPES_##op}; + +FOR_EACH_INTRINSIC(DECL_SAS_FOR_INTRINSIC) +#undef DECL_SAS_FOR_INTRINSIC + +} // namespace wasm +} // namespace js + +#undef _F64 +#undef _F32 +#undef _I32 +#undef _I64 +#undef _PTR +#undef _RoN +#undef _VOID +#undef _END +#undef _Infallible +#undef _FailOnNegI32 +#undef _FailOnNullPtr + +#ifdef DEBUG +ABIArgType ToABIType(FailureMode mode) { + switch (mode) { + case FailureMode::FailOnNegI32: + return ArgType_Int32; + case FailureMode::FailOnNullPtr: + case FailureMode::FailOnInvalidRef: + return ArgType_General; + default: + MOZ_CRASH("unexpected failure mode"); + } +} + +ABIArgType ToABIType(MIRType type) { + switch (type) { + case MIRType::None: + case MIRType::Int32: + return ArgType_Int32; + case MIRType::Int64: + return ArgType_Int64; + case MIRType::Pointer: + case MIRType::RefOrNull: + return ArgType_General; + case MIRType::Float32: + return ArgType_Float32; + case MIRType::Double: + return ArgType_Float64; + default: + MOZ_CRASH("unexpected type"); + } +} + +ABIFunctionType ToABIType(const SymbolicAddressSignature& sig) { + MOZ_ASSERT_IF(sig.failureMode != FailureMode::Infallible, + ToABIType(sig.failureMode) == ToABIType(sig.retType)); + int abiType = ToABIType(sig.retType) << RetType_Shift; + for (int i = 0; i < sig.numArgs; i++) { + abiType |= (ToABIType(sig.argTypes[i]) << (ArgType_Shift * (i + 1))); + } + return ABIFunctionType(abiType); +} +#endif + +// ============================================================================ +// WebAssembly builtin C++ functions called from wasm code to implement internal +// wasm operations: implementations. + +#if defined(JS_CODEGEN_ARM) +extern "C" { + +extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int); + +extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int); +} +#endif + +// This utility function can only be called for builtins that are called +// directly from wasm code. +static JitActivation* CallingActivation(JSContext* cx) { + Activation* act = cx->activation(); + MOZ_ASSERT(act->asJit()->hasWasmExitFP()); + return act->asJit(); +} + +static bool WasmHandleDebugTrap() { + JSContext* cx = TlsContext.get(); // Cold code + JitActivation* activation = CallingActivation(cx); + Frame* fp = activation->wasmExitFP(); + Instance* instance = GetNearestEffectiveInstance(fp); + const Code& code = instance->code(); + MOZ_ASSERT(code.metadata().debugEnabled); + + // The debug trap stub is the innermost frame. It's return address is the + // actual trap site. + const CallSite* site = code.lookupCallSite(fp->returnAddress()); + MOZ_ASSERT(site); + + // Advance to the actual trapping frame. + fp = fp->wasmCaller(); + DebugFrame* debugFrame = DebugFrame::from(fp); + + if (site->kind() == CallSite::EnterFrame) { + if (!instance->debug().enterFrameTrapsEnabled()) { + return true; + } + debugFrame->setIsDebuggee(); + debugFrame->observe(cx); + if (!DebugAPI::onEnterFrame(cx, debugFrame)) { + if (cx->isPropagatingForcedReturn()) { + cx->clearPropagatingForcedReturn(); + // Ignoring forced return because changing code execution order is + // not yet implemented in the wasm baseline. + // TODO properly handle forced return and resume wasm execution. + JS_ReportErrorASCII(cx, + "Unexpected resumption value from onEnterFrame"); + } + return false; + } + return true; + } + if (site->kind() == CallSite::LeaveFrame) { + if (!debugFrame->updateReturnJSValue(cx)) { + return false; + } + bool ok = DebugAPI::onLeaveFrame(cx, debugFrame, nullptr, true); + debugFrame->leave(cx); + return ok; + } + + DebugState& debug = instance->debug(); + MOZ_ASSERT(debug.hasBreakpointTrapAtOffset(site->lineOrBytecode())); + if (debug.stepModeEnabled(debugFrame->funcIndex())) { + if (!DebugAPI::onSingleStep(cx)) { + if (cx->isPropagatingForcedReturn()) { + cx->clearPropagatingForcedReturn(); + // TODO properly handle forced return. + JS_ReportErrorASCII(cx, + "Unexpected resumption value from onSingleStep"); + } + return false; + } + } + if (debug.hasBreakpointSite(site->lineOrBytecode())) { + if (!DebugAPI::onTrap(cx)) { + if (cx->isPropagatingForcedReturn()) { + cx->clearPropagatingForcedReturn(); + // TODO properly handle forced return. + JS_ReportErrorASCII( + cx, "Unexpected resumption value from breakpoint handler"); + } + return false; + } + } + return true; +} + +// Check if the pending exception, if any, is catchable by wasm. +static bool HasCatchableException(JitActivation* activation, JSContext* cx, + MutableHandleValue exn) { + if (!cx->isExceptionPending()) { + return false; + } + + // Traps are generally not catchable as wasm exceptions. The only case in + // which they are catchable is for Trap::ThrowReported, which the wasm + // compiler uses to throw exceptions and is the source of exceptions from C++. + if (activation->isWasmTrapping() && + activation->wasmTrapData().trap != Trap::ThrowReported) { + return false; + } + + if (cx->isThrowingOverRecursed() || cx->isThrowingOutOfMemory()) { + return false; + } + + // Write the exception out here to exn to avoid having to get the pending + // exception and checking for OOM multiple times. + if (cx->getPendingException(exn)) { + // Check if a JS exception originated from a wasm trap. + if (exn.isObject() && exn.toObject().is()) { + ErrorObject& err = exn.toObject().as(); + if (err.fromWasmTrap()) { + return false; + } + } + return true; + } + + MOZ_ASSERT(cx->isThrowingOutOfMemory()); + return false; +} + +// Unwind the entire activation in response to a thrown exception. This function +// is responsible for notifying the debugger of each unwound frame. The return +// value is the new stack address which the calling stub will set to the sp +// register before executing a return instruction. +// +// This function will also look for try-catch handlers and, if not trapping or +// throwing an uncatchable exception, will write the handler info in the return +// argument and return true. +// +// Returns false if a handler isn't found or shouldn't be used (e.g., traps). + +bool wasm::HandleThrow(JSContext* cx, WasmFrameIter& iter, + jit::ResumeFromException* rfe) { + // WasmFrameIter iterates down wasm frames in the activation starting at + // JitActivation::wasmExitFP(). Calling WasmFrameIter::startUnwinding pops + // JitActivation::wasmExitFP() once each time WasmFrameIter is incremented, + // ultimately leaving exit FP null when the WasmFrameIter is done(). This + // is necessary to prevent a DebugFrame from being observed again after we + // just called onLeaveFrame (which would lead to the frame being re-added + // to the map of live frames, right as it becomes trash). + + MOZ_ASSERT(CallingActivation(cx) == iter.activation()); + MOZ_ASSERT(!iter.done()); + iter.setUnwind(WasmFrameIter::Unwind::True); + + // Live wasm code on the stack is kept alive (in TraceJitActivation) by + // marking the instance of every wasm::Frame found by WasmFrameIter. + // However, as explained above, we're popping frames while iterating which + // means that a GC during this loop could collect the code of frames whose + // code is still on the stack. This is actually mostly fine: as soon as we + // return to the throw stub, the entire stack will be popped as a whole, + // returning to the C++ caller. However, we must keep the throw stub alive + // itself which is owned by the innermost instance. + Rooted keepAlive(cx, iter.instance()->object()); + + JitActivation* activation = CallingActivation(cx); + RootedValue exn(cx); + bool hasCatchableException = HasCatchableException(activation, cx, &exn); + + for (; !iter.done(); ++iter) { + // Wasm code can enter same-compartment realms, so reset cx->realm to + // this frame's realm. + cx->setRealmForJitExceptionHandler(iter.instance()->realm()); + + // Only look for an exception handler if there's a catchable exception. + if (hasCatchableException) { + const wasm::Code& code = iter.instance()->code(); + const uint8_t* pc = iter.resumePCinCurrentFrame(); + Tier tier; + const wasm::TryNote* tryNote = code.lookupTryNote((void*)pc, &tier); + + if (tryNote) { + cx->clearPendingException(); + RootedAnyRef ref(cx, AnyRef::null()); + if (!BoxAnyRef(cx, exn, &ref)) { + MOZ_ASSERT(cx->isThrowingOutOfMemory()); + hasCatchableException = false; + continue; + } + + MOZ_ASSERT(iter.instance() == iter.instance()); + iter.instance()->setPendingException(ref); + + rfe->kind = ExceptionResumeKind::WasmCatch; + rfe->framePointer = (uint8_t*)iter.frame(); + rfe->instance = iter.instance(); + + rfe->stackPointer = + (uint8_t*)(rfe->framePointer - tryNote->landingPadFramePushed()); + rfe->target = + iter.instance()->codeBase(tier) + tryNote->landingPadEntryPoint(); + + // Make sure to clear trapping state if we got here due to a trap. + if (activation->isWasmTrapping()) { + activation->finishWasmTrap(); + } + + return true; + } + } + + if (!iter.debugEnabled()) { + continue; + } + + DebugFrame* frame = iter.debugFrame(); + frame->clearReturnJSValue(); + + // Assume ResumeMode::Terminate if no exception is pending -- + // no onExceptionUnwind handlers must be fired. + if (cx->isExceptionPending()) { + if (!DebugAPI::onExceptionUnwind(cx, frame)) { + if (cx->isPropagatingForcedReturn()) { + cx->clearPropagatingForcedReturn(); + // Unexpected trap return -- raising error since throw recovery + // is not yet implemented in the wasm baseline. + // TODO properly handle forced return and resume wasm execution. + JS_ReportErrorASCII( + cx, "Unexpected resumption value from onExceptionUnwind"); + } + } + } + + bool ok = DebugAPI::onLeaveFrame(cx, frame, nullptr, false); + if (ok) { + // Unexpected success from the handler onLeaveFrame -- raising error + // since throw recovery is not yet implemented in the wasm baseline. + // TODO properly handle success and resume wasm execution. + JS_ReportErrorASCII(cx, "Unexpected success from onLeaveFrame"); + } + frame->leave(cx); + } + + MOZ_ASSERT(!cx->activation()->asJit()->isWasmTrapping(), + "unwinding clears the trapping state"); + + // In case of no handler, exit wasm via ret(). + // FailInstanceReg signals to wasm stub to do a failure return. + rfe->kind = ExceptionResumeKind::Wasm; + rfe->framePointer = (uint8_t*)iter.unwoundCallerFP(); + rfe->stackPointer = (uint8_t*)iter.unwoundAddressOfReturnAddress(); + rfe->instance = (Instance*)FailInstanceReg; + rfe->target = nullptr; + return false; +} + +static void* WasmHandleThrow(jit::ResumeFromException* rfe) { + JSContext* cx = TlsContext.get(); // Cold code + JitActivation* activation = CallingActivation(cx); + WasmFrameIter iter(activation); + // We can ignore the return result here because the throw stub code + // can just check the resume kind to see if a handler was found or not. + HandleThrow(cx, iter, rfe); + return rfe; +} + +// Has the same return-value convention as HandleTrap(). +static void* CheckInterrupt(JSContext* cx, JitActivation* activation) { + ResetInterruptState(cx); + + if (!CheckForInterrupt(cx)) { + return nullptr; + } + + void* resumePC = activation->wasmTrapData().resumePC; + activation->finishWasmTrap(); + return resumePC; +} + +// The calling convention between this function and its caller in the stub +// generated by GenerateTrapExit() is: +// - return nullptr if the stub should jump to the throw stub to unwind +// the activation; +// - return the (non-null) resumePC that should be jumped if execution should +// resume after the trap. +static void* WasmHandleTrap() { + JSContext* cx = TlsContext.get(); // Cold code + JitActivation* activation = CallingActivation(cx); + + switch (activation->wasmTrapData().trap) { + case Trap::Unreachable: { + ReportTrapError(cx, JSMSG_WASM_UNREACHABLE); + return nullptr; + } + case Trap::IntegerOverflow: { + ReportTrapError(cx, JSMSG_WASM_INTEGER_OVERFLOW); + return nullptr; + } + case Trap::InvalidConversionToInteger: { + ReportTrapError(cx, JSMSG_WASM_INVALID_CONVERSION); + return nullptr; + } + case Trap::IntegerDivideByZero: { + ReportTrapError(cx, JSMSG_WASM_INT_DIVIDE_BY_ZERO); + return nullptr; + } + case Trap::IndirectCallToNull: { + ReportTrapError(cx, JSMSG_WASM_IND_CALL_TO_NULL); + return nullptr; + } + case Trap::IndirectCallBadSig: { + ReportTrapError(cx, JSMSG_WASM_IND_CALL_BAD_SIG); + return nullptr; + } + case Trap::NullPointerDereference: { + ReportTrapError(cx, JSMSG_WASM_DEREF_NULL); + return nullptr; + } + case Trap::BadCast: { + ReportTrapError(cx, JSMSG_WASM_BAD_CAST); + return nullptr; + } + case Trap::OutOfBounds: { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return nullptr; + } + case Trap::UnalignedAccess: { + ReportTrapError(cx, JSMSG_WASM_UNALIGNED_ACCESS); + return nullptr; + } + case Trap::CheckInterrupt: + return CheckInterrupt(cx, activation); + case Trap::StackOverflow: { + // Instance::setInterrupt() causes a fake stack overflow. Since + // Instance::setInterrupt() is called racily, it's possible for a real + // stack overflow to trap, followed by a racy call to setInterrupt(). + // Thus, we must check for a real stack overflow first before we + // CheckInterrupt() and possibly resume execution. + AutoCheckRecursionLimit recursion(cx); + if (!recursion.check(cx)) { + return nullptr; + } + if (activation->wasmExitInstance()->isInterrupted()) { + return CheckInterrupt(cx, activation); + } + ReportTrapError(cx, JSMSG_OVER_RECURSED); + return nullptr; + } + case Trap::ThrowReported: + // Error was already reported under another name. + return nullptr; + case Trap::Limit: + break; + } + + MOZ_CRASH("unexpected trap"); +} + +static void WasmReportV128JSCall() { + JSContext* cx = TlsContext.get(); // Cold code + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); +} + +static int32_t CoerceInPlace_ToInt32(Value* rawVal) { + JSContext* cx = TlsContext.get(); // Cold code + + int32_t i32; + RootedValue val(cx, *rawVal); + if (!ToInt32(cx, val, &i32)) { + *rawVal = PoisonedObjectValue(0x42); + return false; + } + + *rawVal = Int32Value(i32); + return true; +} + +static int32_t CoerceInPlace_ToBigInt(Value* rawVal) { + JSContext* cx = TlsContext.get(); // Cold code + + RootedValue val(cx, *rawVal); + BigInt* bi = ToBigInt(cx, val); + if (!bi) { + *rawVal = PoisonedObjectValue(0x43); + return false; + } + + *rawVal = BigIntValue(bi); + return true; +} + +static int32_t CoerceInPlace_ToNumber(Value* rawVal) { + JSContext* cx = TlsContext.get(); // Cold code + + double dbl; + RootedValue val(cx, *rawVal); + if (!ToNumber(cx, val, &dbl)) { + *rawVal = PoisonedObjectValue(0x42); + return false; + } + + *rawVal = DoubleValue(dbl); + return true; +} + +static void* BoxValue_Anyref(Value* rawVal) { + JSContext* cx = TlsContext.get(); // Cold code + RootedValue val(cx, *rawVal); + RootedAnyRef result(cx, AnyRef::null()); + if (!BoxAnyRef(cx, val, &result)) { + return nullptr; + } + return result.get().forCompiledCode(); +} + +static int32_t CoerceInPlace_JitEntry(int funcExportIndex, Instance* instance, + Value* argv) { + JSContext* cx = TlsContext.get(); // Cold code + + const Code& code = instance->code(); + const FuncExport& fe = + code.metadata(code.stableTier()).funcExports[funcExportIndex]; + const FuncType& funcType = code.metadata().getFuncExportType(fe); + + for (size_t i = 0; i < funcType.args().length(); i++) { + HandleValue arg = HandleValue::fromMarkedLocation(&argv[i]); + switch (funcType.args()[i].kind()) { + case ValType::I32: { + int32_t i32; + if (!ToInt32(cx, arg, &i32)) { + return false; + } + argv[i] = Int32Value(i32); + break; + } + case ValType::I64: { + // In this case we store a BigInt value as there is no value type + // corresponding directly to an I64. The conversion to I64 happens + // in the JIT entry stub. + BigInt* bigint = ToBigInt(cx, arg); + if (!bigint) { + return false; + } + argv[i] = BigIntValue(bigint); + break; + } + case ValType::F32: + case ValType::F64: { + double dbl; + if (!ToNumber(cx, arg, &dbl)) { + return false; + } + // No need to convert double-to-float for f32, it's done inline + // in the wasm stub later. + argv[i] = DoubleValue(dbl); + break; + } + case ValType::Ref: { + // Guarded against by temporarilyUnsupportedReftypeForEntry() + MOZ_RELEASE_ASSERT(funcType.args()[i].refType().isExtern()); + // Leave Object and Null alone, we will unbox inline. All we need + // to do is convert other values to an Object representation. + if (!arg.isObjectOrNull()) { + RootedAnyRef result(cx, AnyRef::null()); + if (!BoxAnyRef(cx, arg, &result)) { + return false; + } + argv[i].setObject(*result.get().asJSObject()); + } + break; + } + case ValType::V128: { + // Guarded against by hasV128ArgOrRet() + MOZ_CRASH("unexpected input argument in CoerceInPlace_JitEntry"); + } + default: { + MOZ_CRASH("unexpected input argument in CoerceInPlace_JitEntry"); + } + } + } + + return true; +} + +// Allocate a BigInt without GC, corresponds to the similar VMFunction. +static BigInt* AllocateBigIntTenuredNoGC() { + JSContext* cx = TlsContext.get(); // Cold code (the caller is elaborate) + + return cx->newCell(gc::Heap::Tenured); +} + +static int64_t DivI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, + uint32_t y_lo) { + int64_t x = ((uint64_t)x_hi << 32) + x_lo; + int64_t y = ((uint64_t)y_hi << 32) + y_lo; + MOZ_ASSERT(x != INT64_MIN || y != -1); + MOZ_ASSERT(y != 0); + return x / y; +} + +static int64_t UDivI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, + uint32_t y_lo) { + uint64_t x = ((uint64_t)x_hi << 32) + x_lo; + uint64_t y = ((uint64_t)y_hi << 32) + y_lo; + MOZ_ASSERT(y != 0); + return int64_t(x / y); +} + +static int64_t ModI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, + uint32_t y_lo) { + int64_t x = ((uint64_t)x_hi << 32) + x_lo; + int64_t y = ((uint64_t)y_hi << 32) + y_lo; + MOZ_ASSERT(x != INT64_MIN || y != -1); + MOZ_ASSERT(y != 0); + return x % y; +} + +static int64_t UModI64(uint32_t x_hi, uint32_t x_lo, uint32_t y_hi, + uint32_t y_lo) { + uint64_t x = ((uint64_t)x_hi << 32) + x_lo; + uint64_t y = ((uint64_t)y_hi << 32) + y_lo; + MOZ_ASSERT(y != 0); + return int64_t(x % y); +} + +static int64_t TruncateDoubleToInt64(double input) { + // Note: INT64_MAX is not representable in double. It is actually + // INT64_MAX + 1. Therefore also sending the failure value. + if (input >= double(INT64_MAX) || input < double(INT64_MIN) || + std::isnan(input)) { + return int64_t(0x8000000000000000); + } + return int64_t(input); +} + +static uint64_t TruncateDoubleToUint64(double input) { + // Note: UINT64_MAX is not representable in double. It is actually + // UINT64_MAX + 1. Therefore also sending the failure value. + if (input >= double(UINT64_MAX) || input <= -1.0 || std::isnan(input)) { + return int64_t(0x8000000000000000); + } + return uint64_t(input); +} + +static int64_t SaturatingTruncateDoubleToInt64(double input) { + // Handle in-range values (except INT64_MIN). + if (fabs(input) < -double(INT64_MIN)) { + return int64_t(input); + } + // Handle NaN. + if (std::isnan(input)) { + return 0; + } + // Handle positive overflow. + if (input > 0) { + return INT64_MAX; + } + // Handle negative overflow. + return INT64_MIN; +} + +static uint64_t SaturatingTruncateDoubleToUint64(double input) { + // Handle positive overflow. + if (input >= -double(INT64_MIN) * 2.0) { + return UINT64_MAX; + } + // Handle in-range values. + if (input > -1.0) { + return uint64_t(input); + } + // Handle NaN and negative overflow. + return 0; +} + +static double Int64ToDouble(int32_t x_hi, uint32_t x_lo) { + int64_t x = int64_t((uint64_t(x_hi) << 32)) + int64_t(x_lo); + return double(x); +} + +static float Int64ToFloat32(int32_t x_hi, uint32_t x_lo) { + int64_t x = int64_t((uint64_t(x_hi) << 32)) + int64_t(x_lo); + return float(x); +} + +static double Uint64ToDouble(int32_t x_hi, uint32_t x_lo) { + uint64_t x = (uint64_t(x_hi) << 32) + uint64_t(x_lo); + return double(x); +} + +static float Uint64ToFloat32(int32_t x_hi, uint32_t x_lo) { + uint64_t x = (uint64_t(x_hi) << 32) + uint64_t(x_lo); + return float(x); +} + +template +static inline void* FuncCast(F* funcPtr, ABIFunctionType abiType) { + void* pf = JS_FUNC_TO_DATA_PTR(void*, funcPtr); +#ifdef JS_SIMULATOR + pf = Simulator::RedirectNativeFunction(pf, abiType); +#endif + return pf; +} + +#ifdef WASM_CODEGEN_DEBUG +void wasm::PrintI32(int32_t val) { fprintf(stderr, "i32(%d) ", val); } + +void wasm::PrintPtr(uint8_t* val) { fprintf(stderr, "ptr(%p) ", val); } + +void wasm::PrintF32(float val) { fprintf(stderr, "f32(%f) ", val); } + +void wasm::PrintF64(double val) { fprintf(stderr, "f64(%lf) ", val); } + +void wasm::PrintText(const char* out) { fprintf(stderr, "%s", out); } +#endif + +void* wasm::AddressOf(SymbolicAddress imm, ABIFunctionType* abiType) { + // See NeedsBuiltinThunk for a classification of the different names here. + switch (imm) { + case SymbolicAddress::HandleDebugTrap: + *abiType = Args_General0; + return FuncCast(WasmHandleDebugTrap, *abiType); + case SymbolicAddress::HandleThrow: + *abiType = Args_General1; + return FuncCast(WasmHandleThrow, *abiType); + case SymbolicAddress::HandleTrap: + *abiType = Args_General0; + return FuncCast(WasmHandleTrap, *abiType); + case SymbolicAddress::ReportV128JSCall: + *abiType = Args_General0; + return FuncCast(WasmReportV128JSCall, *abiType); + case SymbolicAddress::CallImport_General: + *abiType = Args_Int32_GeneralInt32Int32General; + return FuncCast(Instance::callImport_general, *abiType); + case SymbolicAddress::CoerceInPlace_ToInt32: + *abiType = Args_General1; + return FuncCast(CoerceInPlace_ToInt32, *abiType); + case SymbolicAddress::CoerceInPlace_ToBigInt: + *abiType = Args_General1; + return FuncCast(CoerceInPlace_ToBigInt, *abiType); + case SymbolicAddress::CoerceInPlace_ToNumber: + *abiType = Args_General1; + return FuncCast(CoerceInPlace_ToNumber, *abiType); + case SymbolicAddress::CoerceInPlace_JitEntry: + *abiType = Args_General3; + return FuncCast(CoerceInPlace_JitEntry, *abiType); + case SymbolicAddress::ToInt32: + *abiType = Args_Int_Double; + return FuncCast(JS::ToInt32, *abiType); + case SymbolicAddress::BoxValue_Anyref: + *abiType = Args_General1; + return FuncCast(BoxValue_Anyref, *abiType); + case SymbolicAddress::AllocateBigInt: + *abiType = Args_General0; + return FuncCast(AllocateBigIntTenuredNoGC, *abiType); + case SymbolicAddress::DivI64: + *abiType = Args_General4; + return FuncCast(DivI64, *abiType); + case SymbolicAddress::UDivI64: + *abiType = Args_General4; + return FuncCast(UDivI64, *abiType); + case SymbolicAddress::ModI64: + *abiType = Args_General4; + return FuncCast(ModI64, *abiType); + case SymbolicAddress::UModI64: + *abiType = Args_General4; + return FuncCast(UModI64, *abiType); + case SymbolicAddress::TruncateDoubleToUint64: + *abiType = Args_Int64_Double; + return FuncCast(TruncateDoubleToUint64, *abiType); + case SymbolicAddress::TruncateDoubleToInt64: + *abiType = Args_Int64_Double; + return FuncCast(TruncateDoubleToInt64, *abiType); + case SymbolicAddress::SaturatingTruncateDoubleToUint64: + *abiType = Args_Int64_Double; + return FuncCast(SaturatingTruncateDoubleToUint64, *abiType); + case SymbolicAddress::SaturatingTruncateDoubleToInt64: + *abiType = Args_Int64_Double; + return FuncCast(SaturatingTruncateDoubleToInt64, *abiType); + case SymbolicAddress::Uint64ToDouble: + *abiType = Args_Double_IntInt; + return FuncCast(Uint64ToDouble, *abiType); + case SymbolicAddress::Uint64ToFloat32: + *abiType = Args_Float32_IntInt; + return FuncCast(Uint64ToFloat32, *abiType); + case SymbolicAddress::Int64ToDouble: + *abiType = Args_Double_IntInt; + return FuncCast(Int64ToDouble, *abiType); + case SymbolicAddress::Int64ToFloat32: + *abiType = Args_Float32_IntInt; + return FuncCast(Int64ToFloat32, *abiType); +#if defined(JS_CODEGEN_ARM) + case SymbolicAddress::aeabi_idivmod: + *abiType = Args_General2; + return FuncCast(__aeabi_idivmod, *abiType); + case SymbolicAddress::aeabi_uidivmod: + *abiType = Args_General2; + return FuncCast(__aeabi_uidivmod, *abiType); +#endif + case SymbolicAddress::ModD: + *abiType = Args_Double_DoubleDouble; + return FuncCast(NumberMod, *abiType); + case SymbolicAddress::SinNativeD: + *abiType = Args_Double_Double; + return FuncCast(sin, *abiType); + case SymbolicAddress::SinFdlibmD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::sin, *abiType); + case SymbolicAddress::CosNativeD: + *abiType = Args_Double_Double; + return FuncCast(cos, *abiType); + case SymbolicAddress::CosFdlibmD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::cos, *abiType); + case SymbolicAddress::TanNativeD: + *abiType = Args_Double_Double; + return FuncCast(tan, *abiType); + case SymbolicAddress::TanFdlibmD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::tan, *abiType); + case SymbolicAddress::ASinD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::asin, *abiType); + case SymbolicAddress::ACosD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::acos, *abiType); + case SymbolicAddress::ATanD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::atan, *abiType); + case SymbolicAddress::CeilD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::ceil, *abiType); + case SymbolicAddress::CeilF: + *abiType = Args_Float32_Float32; + return FuncCast(fdlibm::ceilf, *abiType); + case SymbolicAddress::FloorD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::floor, *abiType); + case SymbolicAddress::FloorF: + *abiType = Args_Float32_Float32; + return FuncCast(fdlibm::floorf, *abiType); + case SymbolicAddress::TruncD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::trunc, *abiType); + case SymbolicAddress::TruncF: + *abiType = Args_Float32_Float32; + return FuncCast(fdlibm::truncf, *abiType); + case SymbolicAddress::NearbyIntD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::nearbyint, *abiType); + case SymbolicAddress::NearbyIntF: + *abiType = Args_Float32_Float32; + return FuncCast(fdlibm::nearbyintf, *abiType); + case SymbolicAddress::ExpD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::exp, *abiType); + case SymbolicAddress::LogD: + *abiType = Args_Double_Double; + return FuncCast(fdlibm::log, *abiType); + case SymbolicAddress::PowD: + *abiType = Args_Double_DoubleDouble; + return FuncCast(ecmaPow, *abiType); + case SymbolicAddress::ATan2D: + *abiType = Args_Double_DoubleDouble; + return FuncCast(ecmaAtan2, *abiType); + + case SymbolicAddress::MemoryGrowM32: + *abiType = Args_Int32_GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigMemoryGrowM32)); + return FuncCast(Instance::memoryGrow_m32, *abiType); + case SymbolicAddress::MemoryGrowM64: + *abiType = Args_Int64_GeneralInt64; + MOZ_ASSERT(*abiType == ToABIType(SASigMemoryGrowM64)); + return FuncCast(Instance::memoryGrow_m64, *abiType); + case SymbolicAddress::MemorySizeM32: + *abiType = Args_Int32_General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemorySizeM32)); + return FuncCast(Instance::memorySize_m32, *abiType); + case SymbolicAddress::MemorySizeM64: + *abiType = Args_Int64_General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemorySizeM64)); + return FuncCast(Instance::memorySize_m64, *abiType); + case SymbolicAddress::WaitI32M32: + *abiType = Args_Int32_GeneralInt32Int32Int64; + MOZ_ASSERT(*abiType == ToABIType(SASigWaitI32M32)); + return FuncCast(Instance::wait_i32_m32, *abiType); + case SymbolicAddress::WaitI32M64: + *abiType = Args_Int32_GeneralInt64Int32Int64; + MOZ_ASSERT(*abiType == ToABIType(SASigWaitI32M64)); + return FuncCast(Instance::wait_i32_m64, *abiType); + case SymbolicAddress::WaitI64M32: + *abiType = Args_Int32_GeneralInt32Int64Int64; + MOZ_ASSERT(*abiType == ToABIType(SASigWaitI64M32)); + return FuncCast(Instance::wait_i64_m32, *abiType); + case SymbolicAddress::WaitI64M64: + *abiType = Args_Int32_GeneralInt64Int64Int64; + MOZ_ASSERT(*abiType == ToABIType(SASigWaitI64M64)); + return FuncCast(Instance::wait_i64_m64, *abiType); + case SymbolicAddress::WakeM32: + *abiType = Args_Int32_GeneralInt32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigWakeM32)); + return FuncCast(Instance::wake_m32, *abiType); + case SymbolicAddress::WakeM64: + *abiType = Args_Int32_GeneralInt64Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigWakeM64)); + return FuncCast(Instance::wake_m64, *abiType); + case SymbolicAddress::MemCopyM32: + *abiType = Args_Int32_GeneralInt32Int32Int32General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemCopyM32)); + return FuncCast(Instance::memCopy_m32, *abiType); + case SymbolicAddress::MemCopySharedM32: + *abiType = Args_Int32_GeneralInt32Int32Int32General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemCopySharedM32)); + return FuncCast(Instance::memCopyShared_m32, *abiType); + case SymbolicAddress::MemCopyM64: + *abiType = Args_Int32_GeneralInt64Int64Int64General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemCopyM64)); + return FuncCast(Instance::memCopy_m64, *abiType); + case SymbolicAddress::MemCopySharedM64: + *abiType = Args_Int32_GeneralInt64Int64Int64General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemCopySharedM64)); + return FuncCast(Instance::memCopyShared_m64, *abiType); + case SymbolicAddress::DataDrop: + *abiType = Args_Int32_GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigDataDrop)); + return FuncCast(Instance::dataDrop, *abiType); + case SymbolicAddress::MemFillM32: + *abiType = Args_Int32_GeneralInt32Int32Int32General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemFillM32)); + return FuncCast(Instance::memFill_m32, *abiType); + case SymbolicAddress::MemFillSharedM32: + *abiType = Args_Int32_GeneralInt32Int32Int32General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemFillSharedM32)); + return FuncCast(Instance::memFillShared_m32, *abiType); + case SymbolicAddress::MemFillM64: + *abiType = Args_Int32_GeneralInt64Int32Int64General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemFillM64)); + return FuncCast(Instance::memFill_m64, *abiType); + case SymbolicAddress::MemFillSharedM64: + *abiType = Args_Int32_GeneralInt64Int32Int64General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemFillSharedM64)); + return FuncCast(Instance::memFillShared_m64, *abiType); + case SymbolicAddress::MemDiscardM32: + *abiType = Args_Int32_GeneralInt32Int32General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemDiscardM32)); + return FuncCast(Instance::memDiscard_m32, *abiType); + case SymbolicAddress::MemDiscardSharedM32: + *abiType = Args_Int32_GeneralInt32Int32General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemDiscardSharedM32)); + return FuncCast(Instance::memDiscardShared_m32, *abiType); + case SymbolicAddress::MemDiscardM64: + *abiType = Args_Int32_GeneralInt64Int64General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemDiscardM64)); + return FuncCast(Instance::memDiscard_m64, *abiType); + case SymbolicAddress::MemDiscardSharedM64: + *abiType = Args_Int32_GeneralInt64Int64General; + MOZ_ASSERT(*abiType == ToABIType(SASigMemDiscardSharedM64)); + return FuncCast(Instance::memDiscardShared_m64, *abiType); + case SymbolicAddress::MemInitM32: + *abiType = Args_Int32_GeneralInt32Int32Int32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigMemInitM32)); + return FuncCast(Instance::memInit_m32, *abiType); + case SymbolicAddress::MemInitM64: + *abiType = Args_Int32_GeneralInt64Int32Int32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigMemInitM64)); + return FuncCast(Instance::memInit_m64, *abiType); + case SymbolicAddress::TableCopy: + *abiType = Args_Int32_GeneralInt32Int32Int32Int32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableCopy)); + return FuncCast(Instance::tableCopy, *abiType); + case SymbolicAddress::ElemDrop: + *abiType = Args_Int32_GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigElemDrop)); + return FuncCast(Instance::elemDrop, *abiType); + case SymbolicAddress::TableFill: + *abiType = Args_Int32_GeneralInt32GeneralInt32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableFill)); + return FuncCast(Instance::tableFill, *abiType); + case SymbolicAddress::TableInit: + *abiType = Args_Int32_GeneralInt32Int32Int32Int32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableInit)); + return FuncCast(Instance::tableInit, *abiType); + case SymbolicAddress::TableGet: + *abiType = Args_General_GeneralInt32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableGet)); + return FuncCast(Instance::tableGet, *abiType); + case SymbolicAddress::TableGrow: + *abiType = Args_Int32_GeneralGeneralInt32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableGrow)); + return FuncCast(Instance::tableGrow, *abiType); + case SymbolicAddress::TableSet: + *abiType = Args_Int32_GeneralInt32GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableSet)); + return FuncCast(Instance::tableSet, *abiType); + case SymbolicAddress::TableSize: + *abiType = Args_Int32_GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigTableSize)); + return FuncCast(Instance::tableSize, *abiType); + case SymbolicAddress::RefFunc: + *abiType = Args_General_GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigRefFunc)); + return FuncCast(Instance::refFunc, *abiType); + case SymbolicAddress::PostBarrier: + *abiType = Args_Int32_GeneralGeneral; + MOZ_ASSERT(*abiType == ToABIType(SASigPostBarrier)); + return FuncCast(Instance::postBarrier, *abiType); + case SymbolicAddress::PostBarrierPrecise: + *abiType = Args_Int32_GeneralGeneralGeneral; + MOZ_ASSERT(*abiType == ToABIType(SASigPostBarrierPrecise)); + return FuncCast(Instance::postBarrierPrecise, *abiType); + case SymbolicAddress::PostBarrierPreciseWithOffset: + *abiType = Args_Int32_GeneralGeneralInt32General; + MOZ_ASSERT(*abiType == ToABIType(SASigPostBarrierPreciseWithOffset)); + return FuncCast(Instance::postBarrierPreciseWithOffset, *abiType); + case SymbolicAddress::StructNew: + *abiType = Args_General2; + MOZ_ASSERT(*abiType == ToABIType(SASigStructNew)); + return FuncCast(Instance::structNew, *abiType); + case SymbolicAddress::StructNewUninit: + *abiType = Args_General2; + MOZ_ASSERT(*abiType == ToABIType(SASigStructNewUninit)); + return FuncCast(Instance::structNewUninit, *abiType); + case SymbolicAddress::ArrayNew: + *abiType = Args_General_GeneralInt32General; + MOZ_ASSERT(*abiType == ToABIType(SASigArrayNew)); + return FuncCast(Instance::arrayNew, *abiType); + case SymbolicAddress::ArrayNewUninit: + *abiType = Args_General_GeneralInt32General; + MOZ_ASSERT(*abiType == ToABIType(SASigArrayNewUninit)); + return FuncCast(Instance::arrayNewUninit, *abiType); + case SymbolicAddress::ArrayNewData: + *abiType = Args_General_GeneralInt32Int32GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigArrayNewData)); + return FuncCast(Instance::arrayNewData, *abiType); + case SymbolicAddress::ArrayNewElem: + *abiType = Args_General_GeneralInt32Int32GeneralInt32; + MOZ_ASSERT(*abiType == ToABIType(SASigArrayNewElem)); + return FuncCast(Instance::arrayNewElem, *abiType); + case SymbolicAddress::ArrayCopy: + *abiType = Args_Int32_GeneralGeneralInt32GeneralInt32Int32Int32; + MOZ_ASSERT(*abiType == ToABIType(SASigArrayCopy)); + return FuncCast(Instance::arrayCopy, *abiType); + + case SymbolicAddress::ExceptionNew: + *abiType = Args_General2; + MOZ_ASSERT(*abiType == ToABIType(SASigExceptionNew)); + return FuncCast(Instance::exceptionNew, *abiType); + case SymbolicAddress::ThrowException: + *abiType = Args_Int32_GeneralGeneral; + MOZ_ASSERT(*abiType == ToABIType(SASigThrowException)); + return FuncCast(Instance::throwException, *abiType); + +#ifdef WASM_CODEGEN_DEBUG + case SymbolicAddress::PrintI32: + *abiType = Args_General1; + return FuncCast(PrintI32, *abiType); + case SymbolicAddress::PrintPtr: + *abiType = Args_General1; + return FuncCast(PrintPtr, *abiType); + case SymbolicAddress::PrintF32: + *abiType = Args_Int_Float32; + return FuncCast(PrintF32, *abiType); + case SymbolicAddress::PrintF64: + *abiType = Args_Int_Double; + return FuncCast(PrintF64, *abiType); + case SymbolicAddress::PrintText: + *abiType = Args_General1; + return FuncCast(PrintText, *abiType); +#endif +#define DECL_SAS_TYPE_AND_FN(op, export, sa_name, abitype, entry, idx) \ + case SymbolicAddress::sa_name: \ + *abiType = abitype; \ + return FuncCast(entry, *abiType); + FOR_EACH_INTRINSIC(DECL_SAS_TYPE_AND_FN) +#undef DECL_SAS_TYPE_AND_FN + case SymbolicAddress::Limit: + break; + } + + MOZ_CRASH("Bad SymbolicAddress"); +} + +bool wasm::IsRoundingFunction(SymbolicAddress callee, jit::RoundingMode* mode) { + switch (callee) { + case SymbolicAddress::FloorD: + case SymbolicAddress::FloorF: + *mode = jit::RoundingMode::Down; + return true; + case SymbolicAddress::CeilD: + case SymbolicAddress::CeilF: + *mode = jit::RoundingMode::Up; + return true; + case SymbolicAddress::TruncD: + case SymbolicAddress::TruncF: + *mode = jit::RoundingMode::TowardsZero; + return true; + case SymbolicAddress::NearbyIntD: + case SymbolicAddress::NearbyIntF: + *mode = jit::RoundingMode::NearestTiesToEven; + return true; + default: + return false; + } +} + +bool wasm::NeedsBuiltinThunk(SymbolicAddress sym) { + // Also see "The Wasm Builtin ABIs" in WasmFrame.h. + switch (sym) { + // No thunk, because they do their work within the activation + case SymbolicAddress::HandleThrow: // GenerateThrowStub + case SymbolicAddress::HandleTrap: // GenerateTrapExit + return false; + + // No thunk, because some work has to be done within the activation before + // the activation exit: when called, arbitrary wasm registers are live and + // must be saved, and the stack pointer may not be aligned for any ABI. + case SymbolicAddress::HandleDebugTrap: // GenerateDebugTrapStub + + // No thunk, because their caller manages the activation exit explicitly + case SymbolicAddress::CallImport_General: // GenerateImportInterpExit + case SymbolicAddress::CoerceInPlace_ToInt32: // GenerateImportJitExit + case SymbolicAddress::CoerceInPlace_ToNumber: // GenerateImportJitExit + case SymbolicAddress::CoerceInPlace_ToBigInt: // GenerateImportJitExit + case SymbolicAddress::BoxValue_Anyref: // GenerateImportJitExit + return false; + +#ifdef WASM_CODEGEN_DEBUG + // No thunk, because they call directly into C++ code that does not interact + // with the rest of the VM at all. + case SymbolicAddress::PrintI32: // Debug stub printers + case SymbolicAddress::PrintPtr: + case SymbolicAddress::PrintF32: + case SymbolicAddress::PrintF64: + case SymbolicAddress::PrintText: + return false; +#endif + + // Everyone else gets a thunk to handle the exit from the activation + case SymbolicAddress::ToInt32: + case SymbolicAddress::DivI64: + case SymbolicAddress::UDivI64: + case SymbolicAddress::ModI64: + case SymbolicAddress::UModI64: + case SymbolicAddress::TruncateDoubleToUint64: + case SymbolicAddress::TruncateDoubleToInt64: + case SymbolicAddress::SaturatingTruncateDoubleToUint64: + case SymbolicAddress::SaturatingTruncateDoubleToInt64: + case SymbolicAddress::Uint64ToDouble: + case SymbolicAddress::Uint64ToFloat32: + case SymbolicAddress::Int64ToDouble: + case SymbolicAddress::Int64ToFloat32: +#if defined(JS_CODEGEN_ARM) + case SymbolicAddress::aeabi_idivmod: + case SymbolicAddress::aeabi_uidivmod: +#endif + case SymbolicAddress::AllocateBigInt: + case SymbolicAddress::ModD: + case SymbolicAddress::SinNativeD: + case SymbolicAddress::SinFdlibmD: + case SymbolicAddress::CosNativeD: + case SymbolicAddress::CosFdlibmD: + case SymbolicAddress::TanNativeD: + case SymbolicAddress::TanFdlibmD: + case SymbolicAddress::ASinD: + case SymbolicAddress::ACosD: + case SymbolicAddress::ATanD: + case SymbolicAddress::CeilD: + case SymbolicAddress::CeilF: + case SymbolicAddress::FloorD: + case SymbolicAddress::FloorF: + case SymbolicAddress::TruncD: + case SymbolicAddress::TruncF: + case SymbolicAddress::NearbyIntD: + case SymbolicAddress::NearbyIntF: + case SymbolicAddress::ExpD: + case SymbolicAddress::LogD: + case SymbolicAddress::PowD: + case SymbolicAddress::ATan2D: + case SymbolicAddress::MemoryGrowM32: + case SymbolicAddress::MemoryGrowM64: + case SymbolicAddress::MemorySizeM32: + case SymbolicAddress::MemorySizeM64: + case SymbolicAddress::WaitI32M32: + case SymbolicAddress::WaitI32M64: + case SymbolicAddress::WaitI64M32: + case SymbolicAddress::WaitI64M64: + case SymbolicAddress::WakeM32: + case SymbolicAddress::WakeM64: + case SymbolicAddress::CoerceInPlace_JitEntry: + case SymbolicAddress::ReportV128JSCall: + case SymbolicAddress::MemCopyM32: + case SymbolicAddress::MemCopySharedM32: + case SymbolicAddress::MemCopyM64: + case SymbolicAddress::MemCopySharedM64: + case SymbolicAddress::DataDrop: + case SymbolicAddress::MemFillM32: + case SymbolicAddress::MemFillSharedM32: + case SymbolicAddress::MemFillM64: + case SymbolicAddress::MemFillSharedM64: + case SymbolicAddress::MemDiscardM32: + case SymbolicAddress::MemDiscardSharedM32: + case SymbolicAddress::MemDiscardM64: + case SymbolicAddress::MemDiscardSharedM64: + case SymbolicAddress::MemInitM32: + case SymbolicAddress::MemInitM64: + case SymbolicAddress::TableCopy: + case SymbolicAddress::ElemDrop: + case SymbolicAddress::TableFill: + case SymbolicAddress::TableGet: + case SymbolicAddress::TableGrow: + case SymbolicAddress::TableInit: + case SymbolicAddress::TableSet: + case SymbolicAddress::TableSize: + case SymbolicAddress::RefFunc: + case SymbolicAddress::PostBarrier: + case SymbolicAddress::PostBarrierPrecise: + case SymbolicAddress::PostBarrierPreciseWithOffset: + case SymbolicAddress::ExceptionNew: + case SymbolicAddress::ThrowException: + case SymbolicAddress::StructNew: + case SymbolicAddress::StructNewUninit: + case SymbolicAddress::ArrayNew: + case SymbolicAddress::ArrayNewUninit: + case SymbolicAddress::ArrayNewData: + case SymbolicAddress::ArrayNewElem: + case SymbolicAddress::ArrayCopy: +#define OP(op, export, sa_name, abitype, entry, idx) \ + case SymbolicAddress::sa_name: + FOR_EACH_INTRINSIC(OP) +#undef OP + return true; + + case SymbolicAddress::Limit: + break; + } + + MOZ_CRASH("unexpected symbolic address"); +} + +// ============================================================================ +// [SMDOC] JS Fast Wasm Imports +// +// JS builtins that can be imported by wasm modules and called efficiently +// through thunks. These thunks conform to the internal wasm ABI and thus can be +// patched in for import calls. Calling a JS builtin through a thunk is much +// faster than calling out through the generic import call trampoline which will +// end up in the slowest C++ Instance::callImport path. +// +// Each JS builtin can have several overloads. These must all be enumerated in +// PopulateTypedNatives() so they can be included in the process-wide thunk set. +// Additionally to the traditional overloading based on types, every builtin +// can also have a version implemented by fdlibm or the native math library. +// This is useful for fingerprinting resistance. + +#define FOR_EACH_SIN_COS_TAN_NATIVE(_) \ + _(math_sin, MathSin) \ + _(math_tan, MathTan) \ + _(math_cos, MathCos) + +#define FOR_EACH_UNARY_NATIVE(_) \ + _(math_exp, MathExp) \ + _(math_log, MathLog) \ + _(math_asin, MathASin) \ + _(math_atan, MathATan) \ + _(math_acos, MathACos) \ + _(math_log10, MathLog10) \ + _(math_log2, MathLog2) \ + _(math_log1p, MathLog1P) \ + _(math_expm1, MathExpM1) \ + _(math_sinh, MathSinH) \ + _(math_tanh, MathTanH) \ + _(math_cosh, MathCosH) \ + _(math_asinh, MathASinH) \ + _(math_atanh, MathATanH) \ + _(math_acosh, MathACosH) \ + _(math_sign, MathSign) \ + _(math_trunc, MathTrunc) \ + _(math_cbrt, MathCbrt) + +#define FOR_EACH_BINARY_NATIVE(_) \ + _(ecmaAtan2, MathATan2) \ + _(ecmaHypot, MathHypot) \ + _(ecmaPow, MathPow) + +#define DEFINE_SIN_COS_TAN_FLOAT_WRAPPER(func, _) \ + static float func##_native_impl_f32(float x) { \ + return float(func##_native_impl(double(x))); \ + } \ + static float func##_fdlibm_impl_f32(float x) { \ + return float(func##_fdlibm_impl(double(x))); \ + } + +#define DEFINE_UNARY_FLOAT_WRAPPER(func, _) \ + static float func##_impl_f32(float x) { \ + return float(func##_impl(double(x))); \ + } + +#define DEFINE_BINARY_FLOAT_WRAPPER(func, _) \ + static float func##_f32(float x, float y) { \ + return float(func(double(x), double(y))); \ + } + +FOR_EACH_SIN_COS_TAN_NATIVE(DEFINE_SIN_COS_TAN_FLOAT_WRAPPER) +FOR_EACH_UNARY_NATIVE(DEFINE_UNARY_FLOAT_WRAPPER) +FOR_EACH_BINARY_NATIVE(DEFINE_BINARY_FLOAT_WRAPPER) + +#undef DEFINE_UNARY_FLOAT_WRAPPER +#undef DEFINE_BINARY_FLOAT_WRAPPER + +struct TypedNative { + InlinableNative native; + ABIFunctionType abiType; + enum class FdlibmImpl : uint8_t { No, Yes } fdlibm; + + TypedNative(InlinableNative native, ABIFunctionType abiType, + FdlibmImpl fdlibm) + : native(native), abiType(abiType), fdlibm(fdlibm) {} + + using Lookup = TypedNative; + static HashNumber hash(const Lookup& l) { + return HashGeneric(uint32_t(l.native), uint32_t(l.abiType), + uint32_t(l.fdlibm)); + } + static bool match(const TypedNative& lhs, const Lookup& rhs) { + return lhs.native == rhs.native && lhs.abiType == rhs.abiType && + lhs.fdlibm == rhs.fdlibm; + } +}; + +using TypedNativeToFuncPtrMap = + HashMap; + +static bool PopulateTypedNatives(TypedNativeToFuncPtrMap* typedNatives) { +#define ADD_OVERLOAD(funcName, native, abiType, fdlibm) \ + if (!typedNatives->putNew(TypedNative(InlinableNative::native, abiType, \ + TypedNative::FdlibmImpl::fdlibm), \ + FuncCast(funcName, abiType))) \ + return false; + +#define ADD_SIN_COS_TAN_OVERLOADS(funcName, native) \ + ADD_OVERLOAD(funcName##_native_impl, native, Args_Double_Double, No) \ + ADD_OVERLOAD(funcName##_fdlibm_impl, native, Args_Double_Double, Yes) \ + ADD_OVERLOAD(funcName##_native_impl_f32, native, Args_Float32_Float32, No) \ + ADD_OVERLOAD(funcName##_fdlibm_impl_f32, native, Args_Float32_Float32, Yes) + +#define ADD_UNARY_OVERLOADS(funcName, native) \ + ADD_OVERLOAD(funcName##_impl, native, Args_Double_Double, No) \ + ADD_OVERLOAD(funcName##_impl_f32, native, Args_Float32_Float32, No) + +#define ADD_BINARY_OVERLOADS(funcName, native) \ + ADD_OVERLOAD(funcName, native, Args_Double_DoubleDouble, No) \ + ADD_OVERLOAD(funcName##_f32, native, Args_Float32_Float32Float32, No) + + FOR_EACH_SIN_COS_TAN_NATIVE(ADD_SIN_COS_TAN_OVERLOADS) + FOR_EACH_UNARY_NATIVE(ADD_UNARY_OVERLOADS) + FOR_EACH_BINARY_NATIVE(ADD_BINARY_OVERLOADS) + +#undef ADD_UNARY_OVERLOADS +#undef ADD_BINARY_OVERLOADS + + return true; +} + +#undef FOR_EACH_UNARY_NATIVE +#undef FOR_EACH_BINARY_NATIVE + +// ============================================================================ +// [SMDOC] Process-wide builtin thunk set +// +// Thunks are inserted between wasm calls and the C++ callee and achieve two +// things: +// - bridging the few differences between the internal wasm ABI and the +// external native ABI (viz. float returns on x86 and soft-fp ARM) +// - executing an exit prologue/epilogue which in turn allows any profiling +// iterator to see the full stack up to the wasm operation that called out +// +// Thunks are created for two kinds of C++ callees, enumerated above: +// - SymbolicAddress: for statically compiled calls in the wasm module +// - Imported JS builtins: optimized calls to imports +// +// All thunks are created up front, lazily, when the first wasm module is +// compiled in the process. Thunks are kept alive until the JS engine shuts down +// in the process. No thunks are created at runtime after initialization. This +// simple scheme allows several simplifications: +// - no reference counting to keep thunks alive +// - no problems toggling W^X permissions which, because of multiple executing +// threads, would require each thunk allocation to be on its own page +// The cost for creating all thunks at once is relatively low since all thunks +// fit within the smallest executable-code allocation quantum (64k). + +using TypedNativeToCodeRangeMap = + HashMap; + +using SymbolicAddressToCodeRangeArray = + EnumeratedArray; + +struct BuiltinThunks { + uint8_t* codeBase; + size_t codeSize; + CodeRangeVector codeRanges; + TypedNativeToCodeRangeMap typedNativeToCodeRange; + SymbolicAddressToCodeRangeArray symbolicAddressToCodeRange; + uint32_t provisionalLazyJitEntryOffset; + + BuiltinThunks() : codeBase(nullptr), codeSize(0) {} + + ~BuiltinThunks() { + if (codeBase) { + DeallocateExecutableMemory(codeBase, codeSize); + } + } +}; + +Mutex initBuiltinThunks(mutexid::WasmInitBuiltinThunks); +Atomic builtinThunks; + +bool wasm::EnsureBuiltinThunksInitialized() { + LockGuard guard(initBuiltinThunks); + if (builtinThunks) { + return true; + } + + auto thunks = MakeUnique(); + if (!thunks) { + return false; + } + + LifoAlloc lifo(BUILTIN_THUNK_LIFO_SIZE); + TempAllocator tempAlloc(&lifo); + WasmMacroAssembler masm(tempAlloc); + AutoCreatedBy acb(masm, "wasm::EnsureBuiltinThunksInitialized"); + + for (auto sym : MakeEnumeratedRange(SymbolicAddress::Limit)) { + if (!NeedsBuiltinThunk(sym)) { + thunks->symbolicAddressToCodeRange[sym] = UINT32_MAX; + continue; + } + + uint32_t codeRangeIndex = thunks->codeRanges.length(); + thunks->symbolicAddressToCodeRange[sym] = codeRangeIndex; + + ABIFunctionType abiType; + void* funcPtr = AddressOf(sym, &abiType); + + ExitReason exitReason(sym); + + CallableOffsets offsets; + if (!GenerateBuiltinThunk(masm, abiType, exitReason, funcPtr, &offsets)) { + return false; + } + if (!thunks->codeRanges.emplaceBack(CodeRange::BuiltinThunk, offsets)) { + return false; + } + } + + TypedNativeToFuncPtrMap typedNatives; + if (!PopulateTypedNatives(&typedNatives)) { + return false; + } + + for (TypedNativeToFuncPtrMap::Range r = typedNatives.all(); !r.empty(); + r.popFront()) { + TypedNative typedNative = r.front().key(); + + uint32_t codeRangeIndex = thunks->codeRanges.length(); + if (!thunks->typedNativeToCodeRange.putNew(typedNative, codeRangeIndex)) { + return false; + } + + ABIFunctionType abiType = typedNative.abiType; + void* funcPtr = r.front().value(); + + ExitReason exitReason = ExitReason::Fixed::BuiltinNative; + + CallableOffsets offsets; + if (!GenerateBuiltinThunk(masm, abiType, exitReason, funcPtr, &offsets)) { + return false; + } + if (!thunks->codeRanges.emplaceBack(CodeRange::BuiltinThunk, offsets)) { + return false; + } + } + + // Provisional lazy JitEntry stub: This is a shared stub that can be installed + // in the jit-entry jump table. It uses the JIT ABI and when invoked will + // retrieve (via TlsContext()) and invoke the context-appropriate + // invoke-from-interpreter jit stub, thus serving as the initial, unoptimized + // jit-entry stub for any exported wasm function that has a jit-entry. + +#ifdef DEBUG + // We need to allow this machine code to bake in a C++ code pointer, so we + // disable the wasm restrictions while generating this stub. + JitContext jitContext; + bool oldFlag = jitContext.setIsCompilingWasm(false); +#endif + + Offsets provisionalLazyJitEntryOffsets; + if (!GenerateProvisionalLazyJitEntryStub(masm, + &provisionalLazyJitEntryOffsets)) { + return false; + } + thunks->provisionalLazyJitEntryOffset = provisionalLazyJitEntryOffsets.begin; + +#ifdef DEBUG + jitContext.setIsCompilingWasm(oldFlag); +#endif + + masm.finish(); + if (masm.oom()) { + return false; + } + + size_t allocSize = AlignBytes(masm.bytesNeeded(), ExecutableCodePageSize); + + thunks->codeSize = allocSize; + thunks->codeBase = (uint8_t*)AllocateExecutableMemory( + allocSize, ProtectionSetting::Writable, MemCheckKind::MakeUndefined); + if (!thunks->codeBase) { + return false; + } + + masm.executableCopy(thunks->codeBase); + memset(thunks->codeBase + masm.bytesNeeded(), 0, + allocSize - masm.bytesNeeded()); + + masm.processCodeLabels(thunks->codeBase); + PatchDebugSymbolicAccesses(thunks->codeBase, masm); + + MOZ_ASSERT(masm.callSites().empty()); + MOZ_ASSERT(masm.callSiteTargets().empty()); + MOZ_ASSERT(masm.trapSites().empty()); + MOZ_ASSERT(masm.tryNotes().empty()); + + if (!ExecutableAllocator::makeExecutableAndFlushICache(thunks->codeBase, + thunks->codeSize)) { + return false; + } + + builtinThunks = thunks.release(); + return true; +} + +void wasm::ReleaseBuiltinThunks() { + if (builtinThunks) { + const BuiltinThunks* ptr = builtinThunks; + js_delete(const_cast(ptr)); + builtinThunks = nullptr; + } +} + +void* wasm::SymbolicAddressTarget(SymbolicAddress sym) { + MOZ_ASSERT(builtinThunks); + + ABIFunctionType abiType; + void* funcPtr = AddressOf(sym, &abiType); + + if (!NeedsBuiltinThunk(sym)) { + return funcPtr; + } + + const BuiltinThunks& thunks = *builtinThunks; + uint32_t codeRangeIndex = thunks.symbolicAddressToCodeRange[sym]; + return thunks.codeBase + thunks.codeRanges[codeRangeIndex].begin(); +} + +void* wasm::ProvisionalLazyJitEntryStub() { + MOZ_ASSERT(builtinThunks); + + const BuiltinThunks& thunks = *builtinThunks; + return thunks.codeBase + thunks.provisionalLazyJitEntryOffset; +} + +static Maybe ToBuiltinABIFunctionType( + const FuncType& funcType) { + const ValTypeVector& args = funcType.args(); + const ValTypeVector& results = funcType.results(); + + if (results.length() != 1) { + return Nothing(); + } + + uint32_t abiType; + switch (results[0].kind()) { + case ValType::F32: + abiType = ArgType_Float32 << RetType_Shift; + break; + case ValType::F64: + abiType = ArgType_Float64 << RetType_Shift; + break; + default: + return Nothing(); + } + + if ((args.length() + 1) > (sizeof(uint32_t) * 8 / ArgType_Shift)) { + return Nothing(); + } + + for (size_t i = 0; i < args.length(); i++) { + switch (args[i].kind()) { + case ValType::F32: + abiType |= (ArgType_Float32 << (ArgType_Shift * (i + 1))); + break; + case ValType::F64: + abiType |= (ArgType_Float64 << (ArgType_Shift * (i + 1))); + break; + default: + return Nothing(); + } + } + + return Some(ABIFunctionType(abiType)); +} + +void* wasm::MaybeGetBuiltinThunk(JSFunction* f, const FuncType& funcType) { + MOZ_ASSERT(builtinThunks); + + if (!f->isNativeFun() || !f->hasJitInfo() || + f->jitInfo()->type() != JSJitInfo::InlinableNative) { + return nullptr; + } + + Maybe abiType = ToBuiltinABIFunctionType(funcType); + if (!abiType) { + return nullptr; + } + + const BuiltinThunks& thunks = *builtinThunks; + + // If this function should resist fingerprinting first try to lookup + // the fdlibm version. If that version doesn't exist we still fallback to + // the normal native. + if (math_use_fdlibm_for_sin_cos_tan() || + f->realm()->behaviors().shouldResistFingerprinting()) { + TypedNative typedNative(f->jitInfo()->inlinableNative, *abiType, + TypedNative::FdlibmImpl::Yes); + auto p = + thunks.typedNativeToCodeRange.readonlyThreadsafeLookup(typedNative); + if (p) { + return thunks.codeBase + thunks.codeRanges[p->value()].begin(); + } + } + + TypedNative typedNative(f->jitInfo()->inlinableNative, *abiType, + TypedNative::FdlibmImpl::No); + auto p = thunks.typedNativeToCodeRange.readonlyThreadsafeLookup(typedNative); + if (!p) { + return nullptr; + } + + return thunks.codeBase + thunks.codeRanges[p->value()].begin(); +} + +bool wasm::LookupBuiltinThunk(void* pc, const CodeRange** codeRange, + uint8_t** codeBase) { + if (!builtinThunks) { + return false; + } + + const BuiltinThunks& thunks = *builtinThunks; + if (pc < thunks.codeBase || pc >= thunks.codeBase + thunks.codeSize) { + return false; + } + + *codeBase = thunks.codeBase; + + CodeRange::OffsetInCode target((uint8_t*)pc - thunks.codeBase); + *codeRange = LookupInSorted(thunks.codeRanges, target); + + return !!*codeRange; +} diff --git a/js/src/wasm/WasmBuiltins.h b/js/src/wasm/WasmBuiltins.h new file mode 100644 index 0000000000..0d876f439d --- /dev/null +++ b/js/src/wasm/WasmBuiltins.h @@ -0,0 +1,324 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2017 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_builtins_h +#define wasm_builtins_h + +#include "intgemm/IntegerGemmIntrinsic.h" +#include "jit/IonTypes.h" +#include "wasm/WasmIntrinsicGenerated.h" + +namespace js { +namespace jit { +struct ResumeFromException; +} +namespace wasm { + +class WasmFrameIter; +class CodeRange; +class FuncType; + +// A wasm::SymbolicAddress represents a pointer to a well-known function/global +// that is embedded in wasm code. Since wasm code is serialized and later +// deserialized into a different address space, symbolic addresses must be used +// for *all* pointers into the address space. The MacroAssembler records a list +// of all SymbolicAddresses and the offsets of their use in the code for later +// patching during static linking. + +enum class SymbolicAddress { + ToInt32, +#if defined(JS_CODEGEN_ARM) + aeabi_idivmod, + aeabi_uidivmod, +#endif + ModD, + SinNativeD, + SinFdlibmD, + CosNativeD, + CosFdlibmD, + TanNativeD, + TanFdlibmD, + ASinD, + ACosD, + ATanD, + CeilD, + CeilF, + FloorD, + FloorF, + TruncD, + TruncF, + NearbyIntD, + NearbyIntF, + ExpD, + LogD, + PowD, + ATan2D, + HandleDebugTrap, + HandleThrow, + HandleTrap, + ReportV128JSCall, + CallImport_General, + CoerceInPlace_ToInt32, + CoerceInPlace_ToNumber, + CoerceInPlace_JitEntry, + CoerceInPlace_ToBigInt, + AllocateBigInt, + BoxValue_Anyref, + DivI64, + UDivI64, + ModI64, + UModI64, + TruncateDoubleToInt64, + TruncateDoubleToUint64, + SaturatingTruncateDoubleToInt64, + SaturatingTruncateDoubleToUint64, + Uint64ToFloat32, + Uint64ToDouble, + Int64ToFloat32, + Int64ToDouble, + MemoryGrowM32, + MemoryGrowM64, + MemorySizeM32, + MemorySizeM64, + WaitI32M32, + WaitI32M64, + WaitI64M32, + WaitI64M64, + WakeM32, + WakeM64, + MemCopyM32, + MemCopySharedM32, + MemCopyM64, + MemCopySharedM64, + DataDrop, + MemFillM32, + MemFillSharedM32, + MemFillM64, + MemFillSharedM64, + MemDiscardM32, + MemDiscardSharedM32, + MemDiscardM64, + MemDiscardSharedM64, + MemInitM32, + MemInitM64, + TableCopy, + ElemDrop, + TableFill, + TableGet, + TableGrow, + TableInit, + TableSet, + TableSize, + RefFunc, + PostBarrier, + PostBarrierPrecise, + PostBarrierPreciseWithOffset, + ExceptionNew, + ThrowException, + StructNew, + StructNewUninit, + ArrayNew, + ArrayNewUninit, + ArrayNewData, + ArrayNewElem, + ArrayCopy, +#define DECL_INTRINSIC_SA(op, export, sa_name, abitype, entry, idx) sa_name, + FOR_EACH_INTRINSIC(DECL_INTRINSIC_SA) +#undef DECL_INTRINSIC_SA +#ifdef WASM_CODEGEN_DEBUG + PrintI32, + PrintPtr, + PrintF32, + PrintF64, + PrintText, +#endif + Limit +}; + +// The FailureMode indicates whether, immediately after a call to a builtin +// returns, the return value should be checked against an error condition +// (and if so, which one) which signals that the C++ calle has already +// reported an error and thus wasm needs to wasmTrap(Trap::ThrowReported). + +enum class FailureMode : uint8_t { + Infallible, + FailOnNegI32, + FailOnNullPtr, + FailOnInvalidRef +}; + +// SymbolicAddressSignature carries type information for a function referred +// to by a SymbolicAddress. In order that |argTypes| can be written out as a +// static initialiser, it has to have fixed length. At present +// SymbolicAddressType is used to describe functions with at most 14 arguments, +// so |argTypes| has 15 entries in order to allow the last value to be +// MIRType::None, in the hope of catching any accidental overruns of the +// defined section of the array. + +static constexpr size_t SymbolicAddressSignatureMaxArgs = 14; + +struct SymbolicAddressSignature { + // The SymbolicAddress that is described. + const SymbolicAddress identity; + // The return type, or MIRType::None to denote 'void'. + const jit::MIRType retType; + // The failure mode, which is checked by masm.wasmCallBuiltinInstanceMethod. + const FailureMode failureMode; + // The number of arguments, 0 .. SymbolicAddressSignatureMaxArgs only. + const uint8_t numArgs; + // The argument types; SymbolicAddressSignatureMaxArgs + 1 guard, which + // should be MIRType::None. + const jit::MIRType argTypes[SymbolicAddressSignatureMaxArgs + 1]; +}; + +// The 32 in this assertion is derived as follows: SymbolicAddress is probably +// size-4 aligned-4, but it's at the start of the struct, so there's no +// alignment hole before it. All other components (MIRType and uint8_t) are +// size-1 aligned-1, and there are 18 in total, so it is reasonable to assume +// that they also don't create any alignment holes. Hence it is also +// reasonable to assume that the actual size is 1 * 4 + 18 * 1 == 22. The +// worst-plausible-case rounding will take that up to 32. Hence, the +// assertion uses 32. + +static_assert(sizeof(SymbolicAddressSignature) <= 32, + "SymbolicAddressSignature unexpectedly large"); + +// These provide argument type information for a subset of the SymbolicAddress +// targets, for which type info is needed to generate correct stackmaps. + +extern const SymbolicAddressSignature SASigSinNativeD; +extern const SymbolicAddressSignature SASigSinFdlibmD; +extern const SymbolicAddressSignature SASigCosNativeD; +extern const SymbolicAddressSignature SASigCosFdlibmD; +extern const SymbolicAddressSignature SASigTanNativeD; +extern const SymbolicAddressSignature SASigTanFdlibmD; +extern const SymbolicAddressSignature SASigASinD; +extern const SymbolicAddressSignature SASigACosD; +extern const SymbolicAddressSignature SASigATanD; +extern const SymbolicAddressSignature SASigCeilD; +extern const SymbolicAddressSignature SASigCeilF; +extern const SymbolicAddressSignature SASigFloorD; +extern const SymbolicAddressSignature SASigFloorF; +extern const SymbolicAddressSignature SASigTruncD; +extern const SymbolicAddressSignature SASigTruncF; +extern const SymbolicAddressSignature SASigNearbyIntD; +extern const SymbolicAddressSignature SASigNearbyIntF; +extern const SymbolicAddressSignature SASigExpD; +extern const SymbolicAddressSignature SASigLogD; +extern const SymbolicAddressSignature SASigPowD; +extern const SymbolicAddressSignature SASigATan2D; +extern const SymbolicAddressSignature SASigMemoryGrowM32; +extern const SymbolicAddressSignature SASigMemoryGrowM64; +extern const SymbolicAddressSignature SASigMemorySizeM32; +extern const SymbolicAddressSignature SASigMemorySizeM64; +extern const SymbolicAddressSignature SASigWaitI32M32; +extern const SymbolicAddressSignature SASigWaitI32M64; +extern const SymbolicAddressSignature SASigWaitI64M32; +extern const SymbolicAddressSignature SASigWaitI64M64; +extern const SymbolicAddressSignature SASigWakeM32; +extern const SymbolicAddressSignature SASigWakeM64; +extern const SymbolicAddressSignature SASigMemCopyM32; +extern const SymbolicAddressSignature SASigMemCopySharedM32; +extern const SymbolicAddressSignature SASigMemCopyM64; +extern const SymbolicAddressSignature SASigMemCopySharedM64; +extern const SymbolicAddressSignature SASigDataDrop; +extern const SymbolicAddressSignature SASigMemFillM32; +extern const SymbolicAddressSignature SASigMemFillSharedM32; +extern const SymbolicAddressSignature SASigMemFillM64; +extern const SymbolicAddressSignature SASigMemFillSharedM64; +extern const SymbolicAddressSignature SASigMemDiscardM32; +extern const SymbolicAddressSignature SASigMemDiscardSharedM32; +extern const SymbolicAddressSignature SASigMemDiscardM64; +extern const SymbolicAddressSignature SASigMemDiscardSharedM64; +extern const SymbolicAddressSignature SASigMemInitM32; +extern const SymbolicAddressSignature SASigMemInitM64; +extern const SymbolicAddressSignature SASigTableCopy; +extern const SymbolicAddressSignature SASigElemDrop; +extern const SymbolicAddressSignature SASigTableFill; +extern const SymbolicAddressSignature SASigTableGet; +extern const SymbolicAddressSignature SASigTableGrow; +extern const SymbolicAddressSignature SASigTableInit; +extern const SymbolicAddressSignature SASigTableSet; +extern const SymbolicAddressSignature SASigTableSize; +extern const SymbolicAddressSignature SASigRefFunc; +extern const SymbolicAddressSignature SASigPostBarrier; +extern const SymbolicAddressSignature SASigPostBarrierPrecise; +extern const SymbolicAddressSignature SASigPostBarrierPreciseWithOffset; +extern const SymbolicAddressSignature SASigExceptionNew; +extern const SymbolicAddressSignature SASigThrowException; +extern const SymbolicAddressSignature SASigStructNew; +extern const SymbolicAddressSignature SASigStructNewUninit; +extern const SymbolicAddressSignature SASigArrayNew; +extern const SymbolicAddressSignature SASigArrayNewUninit; +extern const SymbolicAddressSignature SASigArrayNewData; +extern const SymbolicAddressSignature SASigArrayNewElem; +extern const SymbolicAddressSignature SASigArrayCopy; +#define EXT_INTR_SA_DECL(op, export, sa_name, abitype, entry, idx) \ + extern const SymbolicAddressSignature SASig##sa_name; +FOR_EACH_INTRINSIC(EXT_INTR_SA_DECL) +#undef EXT_INTR_SA_DECL + +bool IsRoundingFunction(SymbolicAddress callee, jit::RoundingMode* mode); + +// A SymbolicAddress that NeedsBuiltinThunk() will call through a thunk to the +// C++ function. This will be true for all normal calls from normal wasm +// function code. Only calls to C++ from other exits/thunks do not need a thunk. +// See "The Wasm-builtin ABIs in WasmFrame.h". + +bool NeedsBuiltinThunk(SymbolicAddress sym); + +// This function queries whether pc is in one of the process's builtin thunks +// and, if so, returns the CodeRange and pointer to the code segment that the +// CodeRange is relative to. + +bool LookupBuiltinThunk(void* pc, const CodeRange** codeRange, + uint8_t** codeBase); + +// EnsureBuiltinThunksInitialized() must be called, and must succeed, before +// SymbolicAddressTarget() or MaybeGetBuiltinThunk(). This function creates all +// thunks for the process. ReleaseBuiltinThunks() should be called before +// ReleaseProcessExecutableMemory() so that the latter can assert that all +// executable code has been released. + +bool EnsureBuiltinThunksInitialized(); + +bool HandleThrow(JSContext* cx, WasmFrameIter& iter, + jit::ResumeFromException* rfe); + +void* SymbolicAddressTarget(SymbolicAddress sym); + +void* ProvisionalLazyJitEntryStub(); + +void* MaybeGetBuiltinThunk(JSFunction* f, const FuncType& funcType); + +void ReleaseBuiltinThunks(); + +void* AddressOf(SymbolicAddress imm, jit::ABIFunctionType* abiType); + +#ifdef WASM_CODEGEN_DEBUG +void PrintI32(int32_t val); +void PrintF32(float val); +void PrintF64(double val); +void PrintPtr(uint8_t* val); +void PrintText(const char* out); +#endif + +} // namespace wasm +} // namespace js + +#endif // wasm_builtins_h diff --git a/js/src/wasm/WasmCode.cpp b/js/src/wasm/WasmCode.cpp new file mode 100644 index 0000000000..063d2a020c --- /dev/null +++ b/js/src/wasm/WasmCode.cpp @@ -0,0 +1,1253 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmCode.h" + +#include "mozilla/Atomics.h" +#include "mozilla/BinarySearch.h" +#include "mozilla/EnumeratedRange.h" +#include "mozilla/Sprintf.h" + +#include + +#include "jsnum.h" + +#include "jit/Disassemble.h" +#include "jit/ExecutableAllocator.h" +#include "jit/MacroAssembler.h" +#include "jit/PerfSpewer.h" +#include "util/Poison.h" +#ifdef MOZ_VTUNE +# include "vtune/VTuneWrapper.h" +#endif +#include "wasm/WasmModule.h" +#include "wasm/WasmProcess.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmUtility.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; +using mozilla::BinarySearch; +using mozilla::BinarySearchIf; +using mozilla::MakeEnumeratedRange; +using mozilla::PodAssign; + +size_t LinkData::SymbolicLinkArray::sizeOfExcludingThis( + MallocSizeOf mallocSizeOf) const { + size_t size = 0; + for (const Uint32Vector& offsets : *this) { + size += offsets.sizeOfExcludingThis(mallocSizeOf); + } + return size; +} + +CodeSegment::~CodeSegment() { + if (unregisterOnDestroy_) { + UnregisterCodeSegment(this); + } +} + +static uint32_t RoundupCodeLength(uint32_t codeLength) { + // AllocateExecutableMemory() requires a multiple of ExecutableCodePageSize. + return RoundUp(codeLength, ExecutableCodePageSize); +} + +UniqueCodeBytes wasm::AllocateCodeBytes(uint32_t codeLength) { + if (codeLength > MaxCodeBytesPerProcess) { + return nullptr; + } + + static_assert(MaxCodeBytesPerProcess <= INT32_MAX, "rounding won't overflow"); + uint32_t roundedCodeLength = RoundupCodeLength(codeLength); + + void* p = + AllocateExecutableMemory(roundedCodeLength, ProtectionSetting::Writable, + MemCheckKind::MakeUndefined); + + // If the allocation failed and the embedding gives us a last-ditch attempt + // to purge all memory (which, in gecko, does a purging GC/CC/GC), do that + // then retry the allocation. + if (!p) { + if (OnLargeAllocationFailure) { + OnLargeAllocationFailure(); + p = AllocateExecutableMemory(roundedCodeLength, + ProtectionSetting::Writable, + MemCheckKind::MakeUndefined); + } + } + + if (!p) { + return nullptr; + } + + // Zero the padding. + memset(((uint8_t*)p) + codeLength, 0, roundedCodeLength - codeLength); + + // We account for the bytes allocated in WasmModuleObject::create, where we + // have the necessary JSContext. + + return UniqueCodeBytes((uint8_t*)p, FreeCode(roundedCodeLength)); +} + +bool CodeSegment::initialize(const CodeTier& codeTier) { + MOZ_ASSERT(!initialized()); + codeTier_ = &codeTier; + MOZ_ASSERT(initialized()); + + // In the case of tiering, RegisterCodeSegment() immediately makes this code + // segment live to access from other threads executing the containing + // module. So only call once the CodeSegment is fully initialized. + if (!RegisterCodeSegment(this)) { + return false; + } + + // This bool is only used by the destructor which cannot be called racily + // and so it is not a problem to mutate it after RegisterCodeSegment(). + MOZ_ASSERT(!unregisterOnDestroy_); + unregisterOnDestroy_ = true; + return true; +} + +const Code& CodeSegment::code() const { + MOZ_ASSERT(codeTier_); + return codeTier_->code(); +} + +void CodeSegment::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code) const { + *code += RoundupCodeLength(length()); +} + +void FreeCode::operator()(uint8_t* bytes) { + MOZ_ASSERT(codeLength); + MOZ_ASSERT(codeLength == RoundupCodeLength(codeLength)); + +#ifdef MOZ_VTUNE + vtune::UnmarkBytes(bytes, codeLength); +#endif + DeallocateExecutableMemory(bytes, codeLength); +} + +bool wasm::StaticallyLink(const ModuleSegment& ms, const LinkData& linkData) { + for (LinkData::InternalLink link : linkData.internalLinks) { + CodeLabel label; + label.patchAt()->bind(link.patchAtOffset); + label.target()->bind(link.targetOffset); +#ifdef JS_CODELABEL_LINKMODE + label.setLinkMode(static_cast(link.mode)); +#endif + Assembler::Bind(ms.base(), label); + } + + if (!EnsureBuiltinThunksInitialized()) { + return false; + } + + for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) { + const Uint32Vector& offsets = linkData.symbolicLinks[imm]; + if (offsets.empty()) { + continue; + } + + void* target = SymbolicAddressTarget(imm); + for (uint32_t offset : offsets) { + uint8_t* patchAt = ms.base() + offset; + Assembler::PatchDataWithValueCheck(CodeLocationLabel(patchAt), + PatchedImmPtr(target), + PatchedImmPtr((void*)-1)); + } + } + + return true; +} + +void wasm::StaticallyUnlink(uint8_t* base, const LinkData& linkData) { + for (LinkData::InternalLink link : linkData.internalLinks) { + CodeLabel label; + label.patchAt()->bind(link.patchAtOffset); + label.target()->bind(-size_t(base)); // to reset immediate to null +#ifdef JS_CODELABEL_LINKMODE + label.setLinkMode(static_cast(link.mode)); +#endif + Assembler::Bind(base, label); + } + + for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) { + const Uint32Vector& offsets = linkData.symbolicLinks[imm]; + if (offsets.empty()) { + continue; + } + + void* target = SymbolicAddressTarget(imm); + for (uint32_t offset : offsets) { + uint8_t* patchAt = base + offset; + Assembler::PatchDataWithValueCheck(CodeLocationLabel(patchAt), + PatchedImmPtr((void*)-1), + PatchedImmPtr(target)); + } + } +} + +static bool AppendToString(const char* str, UTF8Bytes* bytes) { + return bytes->append(str, strlen(str)) && bytes->append('\0'); +} + +static void SendCodeRangesToProfiler(const ModuleSegment& ms, + const Metadata& metadata, + const CodeRangeVector& codeRanges) { + bool enabled = false; + enabled |= PerfEnabled(); +#ifdef MOZ_VTUNE + enabled |= vtune::IsProfilingActive(); +#endif + if (!enabled) { + return; + } + + for (const CodeRange& codeRange : codeRanges) { + if (!codeRange.hasFuncIndex()) { + continue; + } + + uintptr_t start = uintptr_t(ms.base() + codeRange.begin()); + uintptr_t size = codeRange.end() - codeRange.begin(); + + UTF8Bytes name; + if (!metadata.getFuncNameStandalone(codeRange.funcIndex(), &name)) { + return; + } + + // Avoid "unused" warnings + (void)start; + (void)size; + + if (PerfEnabled()) { + const char* file = metadata.filename.get(); + if (codeRange.isFunction()) { + if (!name.append('\0')) { + return; + } + unsigned line = codeRange.funcLineOrBytecode(); + CollectPerfSpewerWasmFunctionMap(start, size, file, line, name.begin()); + } else if (codeRange.isInterpEntry()) { + if (!AppendToString(" slow entry", &name)) { + return; + } + CollectPerfSpewerWasmMap(start, size, file, name.begin()); + } else if (codeRange.isJitEntry()) { + if (!AppendToString(" fast entry", &name)) { + return; + } + CollectPerfSpewerWasmMap(start, size, file, name.begin()); + } else if (codeRange.isImportInterpExit()) { + if (!AppendToString(" slow exit", &name)) { + return; + } + CollectPerfSpewerWasmMap(start, size, file, name.begin()); + } else if (codeRange.isImportJitExit()) { + if (!AppendToString(" fast exit", &name)) { + return; + } + CollectPerfSpewerWasmMap(start, size, file, name.begin()); + } else { + MOZ_CRASH("unhandled perf hasFuncIndex type"); + } + } +#ifdef MOZ_VTUNE + if (!vtune::IsProfilingActive()) { + continue; + } + if (!codeRange.isFunction()) { + continue; + } + if (!name.append('\0')) { + return; + } + vtune::MarkWasm(vtune::GenerateUniqueMethodID(), name.begin(), (void*)start, + size); +#endif + } +} + +ModuleSegment::ModuleSegment(Tier tier, UniqueCodeBytes codeBytes, + uint32_t codeLength, const LinkData& linkData) + : CodeSegment(std::move(codeBytes), codeLength, CodeSegment::Kind::Module), + tier_(tier), + trapCode_(base() + linkData.trapOffset) {} + +/* static */ +UniqueModuleSegment ModuleSegment::create(Tier tier, MacroAssembler& masm, + const LinkData& linkData) { + uint32_t codeLength = masm.bytesNeeded(); + + UniqueCodeBytes codeBytes = AllocateCodeBytes(codeLength); + if (!codeBytes) { + return nullptr; + } + + masm.executableCopy(codeBytes.get()); + + return js::MakeUnique(tier, std::move(codeBytes), codeLength, + linkData); +} + +/* static */ +UniqueModuleSegment ModuleSegment::create(Tier tier, const Bytes& unlinkedBytes, + const LinkData& linkData) { + uint32_t codeLength = unlinkedBytes.length(); + + UniqueCodeBytes codeBytes = AllocateCodeBytes(codeLength); + if (!codeBytes) { + return nullptr; + } + + memcpy(codeBytes.get(), unlinkedBytes.begin(), codeLength); + + return js::MakeUnique(tier, std::move(codeBytes), codeLength, + linkData); +} + +bool ModuleSegment::initialize(const CodeTier& codeTier, + const LinkData& linkData, + const Metadata& metadata, + const MetadataTier& metadataTier) { + if (!StaticallyLink(*this, linkData)) { + return false; + } + + // Optimized compilation finishes on a background thread, so we must make sure + // to flush the icaches of all the executing threads. + // Reprotect the whole region to avoid having separate RW and RX mappings. + if (!ExecutableAllocator::makeExecutableAndFlushICache( + base(), RoundupCodeLength(length()))) { + return false; + } + + SendCodeRangesToProfiler(*this, metadata, metadataTier.codeRanges); + + // See comments in CodeSegment::initialize() for why this must be last. + return CodeSegment::initialize(codeTier); +} + +void ModuleSegment::addSizeOfMisc(mozilla::MallocSizeOf mallocSizeOf, + size_t* code, size_t* data) const { + CodeSegment::addSizeOfMisc(mallocSizeOf, code); + *data += mallocSizeOf(this); +} + +const CodeRange* ModuleSegment::lookupRange(const void* pc) const { + return codeTier().lookupRange(pc); +} + +size_t CacheableChars::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return mallocSizeOf(get()); +} + +size_t MetadataTier::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return funcToCodeRange.sizeOfExcludingThis(mallocSizeOf) + + codeRanges.sizeOfExcludingThis(mallocSizeOf) + + callSites.sizeOfExcludingThis(mallocSizeOf) + + tryNotes.sizeOfExcludingThis(mallocSizeOf) + + trapSites.sizeOfExcludingThis(mallocSizeOf) + + funcImports.sizeOfExcludingThis(mallocSizeOf) + + funcExports.sizeOfExcludingThis(mallocSizeOf); +} + +UniqueLazyStubSegment LazyStubSegment::create(const CodeTier& codeTier, + size_t length) { + UniqueCodeBytes codeBytes = AllocateCodeBytes(length); + if (!codeBytes) { + return nullptr; + } + + auto segment = js::MakeUnique(std::move(codeBytes), length); + if (!segment || !segment->initialize(codeTier)) { + return nullptr; + } + + return segment; +} + +bool LazyStubSegment::hasSpace(size_t bytes) const { + MOZ_ASSERT(AlignBytesNeeded(bytes) == bytes); + return bytes <= length() && usedBytes_ <= length() - bytes; +} + +bool LazyStubSegment::addStubs(const Metadata& metadata, size_t codeLength, + const Uint32Vector& funcExportIndices, + const FuncExportVector& funcExports, + const CodeRangeVector& codeRanges, + uint8_t** codePtr, + size_t* indexFirstInsertedCodeRange) { + MOZ_ASSERT(hasSpace(codeLength)); + + size_t offsetInSegment = usedBytes_; + *codePtr = base() + usedBytes_; + usedBytes_ += codeLength; + + *indexFirstInsertedCodeRange = codeRanges_.length(); + + if (!codeRanges_.reserve(codeRanges_.length() + 2 * codeRanges.length())) { + return false; + } + + size_t i = 0; + for (uint32_t funcExportIndex : funcExportIndices) { + const FuncExport& fe = funcExports[funcExportIndex]; + const FuncType& funcType = metadata.getFuncExportType(fe); + const CodeRange& interpRange = codeRanges[i]; + MOZ_ASSERT(interpRange.isInterpEntry()); + MOZ_ASSERT(interpRange.funcIndex() == + funcExports[funcExportIndex].funcIndex()); + + codeRanges_.infallibleAppend(interpRange); + codeRanges_.back().offsetBy(offsetInSegment); + i++; + + if (!funcType.canHaveJitEntry()) { + continue; + } + + const CodeRange& jitRange = codeRanges[i]; + MOZ_ASSERT(jitRange.isJitEntry()); + MOZ_ASSERT(jitRange.funcIndex() == interpRange.funcIndex()); + + codeRanges_.infallibleAppend(jitRange); + codeRanges_.back().offsetBy(offsetInSegment); + i++; + } + + return true; +} + +const CodeRange* LazyStubSegment::lookupRange(const void* pc) const { + // Do not search if the search will not find anything. There can be many + // segments, each with many entries. + if (pc < base() || pc >= base() + length()) { + return nullptr; + } + return LookupInSorted(codeRanges_, + CodeRange::OffsetInCode((uint8_t*)pc - base())); +} + +void LazyStubSegment::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const { + CodeSegment::addSizeOfMisc(mallocSizeOf, code); + *data += codeRanges_.sizeOfExcludingThis(mallocSizeOf); + *data += mallocSizeOf(this); +} + +// When allocating a single stub to a page, we should not always place the stub +// at the beginning of the page as the stubs will tend to thrash the icache by +// creating conflicts (everything ends up in the same cache set). Instead, +// locate stubs at different line offsets up to 3/4 the system page size (the +// code allocation quantum). +// +// This may be called on background threads, hence the atomic. + +static void PadCodeForSingleStub(MacroAssembler& masm) { + // Assume 64B icache line size + static uint8_t zeroes[64]; + + // The counter serves only to spread the code out, it has no other meaning and + // can wrap around. + static mozilla::Atomic + counter(0); + + uint32_t maxPadLines = ((gc::SystemPageSize() * 3) / 4) / sizeof(zeroes); + uint32_t padLines = counter++ % maxPadLines; + for (uint32_t i = 0; i < padLines; i++) { + masm.appendRawCode(zeroes, sizeof(zeroes)); + } +} + +static constexpr unsigned LAZY_STUB_LIFO_DEFAULT_CHUNK_SIZE = 8 * 1024; + +bool LazyStubTier::createManyEntryStubs(const Uint32Vector& funcExportIndices, + const Metadata& metadata, + const CodeTier& codeTier, + size_t* stubSegmentIndex) { + MOZ_ASSERT(funcExportIndices.length()); + + LifoAlloc lifo(LAZY_STUB_LIFO_DEFAULT_CHUNK_SIZE); + TempAllocator alloc(&lifo); + JitContext jitContext; + WasmMacroAssembler masm(alloc); + + if (funcExportIndices.length() == 1) { + PadCodeForSingleStub(masm); + } + + const MetadataTier& metadataTier = codeTier.metadata(); + const FuncExportVector& funcExports = metadataTier.funcExports; + uint8_t* moduleSegmentBase = codeTier.segment().base(); + + CodeRangeVector codeRanges; + DebugOnly numExpectedRanges = 0; + for (uint32_t funcExportIndex : funcExportIndices) { + const FuncExport& fe = funcExports[funcExportIndex]; + const FuncType& funcType = metadata.getFuncExportType(fe); + // Exports that don't support a jit entry get only the interp entry. + numExpectedRanges += (funcType.canHaveJitEntry() ? 2 : 1); + void* calleePtr = + moduleSegmentBase + metadataTier.codeRange(fe).funcUncheckedCallEntry(); + Maybe callee; + callee.emplace(calleePtr, ImmPtr::NoCheckToken()); + if (!GenerateEntryStubs(masm, funcExportIndex, fe, funcType, callee, + /* asmjs */ false, &codeRanges)) { + return false; + } + } + MOZ_ASSERT(codeRanges.length() == numExpectedRanges, + "incorrect number of entries per function"); + + masm.finish(); + + MOZ_ASSERT(masm.callSites().empty()); + MOZ_ASSERT(masm.callSiteTargets().empty()); + MOZ_ASSERT(masm.trapSites().empty()); + MOZ_ASSERT(masm.tryNotes().empty()); + + if (masm.oom()) { + return false; + } + + size_t codeLength = LazyStubSegment::AlignBytesNeeded(masm.bytesNeeded()); + + if (!stubSegments_.length() || + !stubSegments_[lastStubSegmentIndex_]->hasSpace(codeLength)) { + size_t newSegmentSize = std::max(codeLength, ExecutableCodePageSize); + UniqueLazyStubSegment newSegment = + LazyStubSegment::create(codeTier, newSegmentSize); + if (!newSegment) { + return false; + } + lastStubSegmentIndex_ = stubSegments_.length(); + if (!stubSegments_.emplaceBack(std::move(newSegment))) { + return false; + } + } + + LazyStubSegment* segment = stubSegments_[lastStubSegmentIndex_].get(); + *stubSegmentIndex = lastStubSegmentIndex_; + + size_t interpRangeIndex; + uint8_t* codePtr = nullptr; + if (!segment->addStubs(metadata, codeLength, funcExportIndices, funcExports, + codeRanges, &codePtr, &interpRangeIndex)) { + return false; + } + + masm.executableCopy(codePtr); + PatchDebugSymbolicAccesses(codePtr, masm); + memset(codePtr + masm.bytesNeeded(), 0, codeLength - masm.bytesNeeded()); + + for (const CodeLabel& label : masm.codeLabels()) { + Assembler::Bind(codePtr, label); + } + + if (!ExecutableAllocator::makeExecutableAndFlushICache(codePtr, codeLength)) { + return false; + } + + // Create lazy function exports for funcIndex -> entry lookup. + if (!exports_.reserve(exports_.length() + funcExportIndices.length())) { + return false; + } + + for (uint32_t funcExportIndex : funcExportIndices) { + const FuncExport& fe = funcExports[funcExportIndex]; + const FuncType& funcType = metadata.getFuncExportType(fe); + + DebugOnly cr = segment->codeRanges()[interpRangeIndex]; + MOZ_ASSERT(cr.value.isInterpEntry()); + MOZ_ASSERT(cr.value.funcIndex() == fe.funcIndex()); + + LazyFuncExport lazyExport(fe.funcIndex(), *stubSegmentIndex, + interpRangeIndex); + + size_t exportIndex; + const uint32_t targetFunctionIndex = fe.funcIndex(); + MOZ_ALWAYS_FALSE(BinarySearchIf( + exports_, 0, exports_.length(), + [targetFunctionIndex](const LazyFuncExport& funcExport) { + return targetFunctionIndex - funcExport.funcIndex; + }, + &exportIndex)); + MOZ_ALWAYS_TRUE( + exports_.insert(exports_.begin() + exportIndex, std::move(lazyExport))); + + // Exports that don't support a jit entry get only the interp entry. + interpRangeIndex += (funcType.canHaveJitEntry() ? 2 : 1); + } + + return true; +} + +bool LazyStubTier::createOneEntryStub(uint32_t funcExportIndex, + const Metadata& metadata, + const CodeTier& codeTier) { + Uint32Vector funcExportIndexes; + if (!funcExportIndexes.append(funcExportIndex)) { + return false; + } + + size_t stubSegmentIndex; + if (!createManyEntryStubs(funcExportIndexes, metadata, codeTier, + &stubSegmentIndex)) { + return false; + } + + const UniqueLazyStubSegment& segment = stubSegments_[stubSegmentIndex]; + const CodeRangeVector& codeRanges = segment->codeRanges(); + + const FuncExport& fe = codeTier.metadata().funcExports[funcExportIndex]; + const FuncType& funcType = metadata.getFuncExportType(fe); + + // Exports that don't support a jit entry get only the interp entry. + if (!funcType.canHaveJitEntry()) { + MOZ_ASSERT(codeRanges.length() >= 1); + MOZ_ASSERT(codeRanges.back().isInterpEntry()); + return true; + } + + MOZ_ASSERT(codeRanges.length() >= 2); + MOZ_ASSERT(codeRanges[codeRanges.length() - 2].isInterpEntry()); + + const CodeRange& cr = codeRanges[codeRanges.length() - 1]; + MOZ_ASSERT(cr.isJitEntry()); + + codeTier.code().setJitEntry(cr.funcIndex(), segment->base() + cr.begin()); + return true; +} + +bool LazyStubTier::createTier2(const Uint32Vector& funcExportIndices, + const Metadata& metadata, + const CodeTier& codeTier, + Maybe* outStubSegmentIndex) { + if (!funcExportIndices.length()) { + return true; + } + + size_t stubSegmentIndex; + if (!createManyEntryStubs(funcExportIndices, metadata, codeTier, + &stubSegmentIndex)) { + return false; + } + + outStubSegmentIndex->emplace(stubSegmentIndex); + return true; +} + +void LazyStubTier::setJitEntries(const Maybe& stubSegmentIndex, + const Code& code) { + if (!stubSegmentIndex) { + return; + } + const UniqueLazyStubSegment& segment = stubSegments_[*stubSegmentIndex]; + for (const CodeRange& cr : segment->codeRanges()) { + if (!cr.isJitEntry()) { + continue; + } + code.setJitEntry(cr.funcIndex(), segment->base() + cr.begin()); + } +} + +bool LazyStubTier::hasEntryStub(uint32_t funcIndex) const { + size_t match; + return BinarySearchIf( + exports_, 0, exports_.length(), + [funcIndex](const LazyFuncExport& funcExport) { + return funcIndex - funcExport.funcIndex; + }, + &match); +} + +void* LazyStubTier::lookupInterpEntry(uint32_t funcIndex) const { + size_t match; + if (!BinarySearchIf( + exports_, 0, exports_.length(), + [funcIndex](const LazyFuncExport& funcExport) { + return funcIndex - funcExport.funcIndex; + }, + &match)) { + return nullptr; + } + const LazyFuncExport& fe = exports_[match]; + const LazyStubSegment& stub = *stubSegments_[fe.lazyStubSegmentIndex]; + return stub.base() + stub.codeRanges()[fe.funcCodeRangeIndex].begin(); +} + +void LazyStubTier::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const { + *data += sizeof(*this); + *data += exports_.sizeOfExcludingThis(mallocSizeOf); + for (const UniqueLazyStubSegment& stub : stubSegments_) { + stub->addSizeOfMisc(mallocSizeOf, code, data); + } +} + +size_t Metadata::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return types->sizeOfExcludingThis(mallocSizeOf) + + globals.sizeOfExcludingThis(mallocSizeOf) + + tables.sizeOfExcludingThis(mallocSizeOf) + + tags.sizeOfExcludingThis(mallocSizeOf) + + funcNames.sizeOfExcludingThis(mallocSizeOf) + + filename.sizeOfExcludingThis(mallocSizeOf) + + sourceMapURL.sizeOfExcludingThis(mallocSizeOf); +} + +struct ProjectFuncIndex { + const FuncExportVector& funcExports; + explicit ProjectFuncIndex(const FuncExportVector& funcExports) + : funcExports(funcExports) {} + uint32_t operator[](size_t index) const { + return funcExports[index].funcIndex(); + } +}; + +FuncExport& MetadataTier::lookupFuncExport( + uint32_t funcIndex, size_t* funcExportIndex /* = nullptr */) { + size_t match; + if (!BinarySearch(ProjectFuncIndex(funcExports), 0, funcExports.length(), + funcIndex, &match)) { + MOZ_CRASH("missing function export"); + } + if (funcExportIndex) { + *funcExportIndex = match; + } + return funcExports[match]; +} + +const FuncExport& MetadataTier::lookupFuncExport( + uint32_t funcIndex, size_t* funcExportIndex) const { + return const_cast(this)->lookupFuncExport(funcIndex, + funcExportIndex); +} + +static bool AppendName(const Bytes& namePayload, const Name& name, + UTF8Bytes* bytes) { + MOZ_RELEASE_ASSERT(name.offsetInNamePayload <= namePayload.length()); + MOZ_RELEASE_ASSERT(name.length <= + namePayload.length() - name.offsetInNamePayload); + return bytes->append( + (const char*)namePayload.begin() + name.offsetInNamePayload, name.length); +} + +static bool AppendFunctionIndexName(uint32_t funcIndex, UTF8Bytes* bytes) { + const char beforeFuncIndex[] = "wasm-function["; + const char afterFuncIndex[] = "]"; + + Int32ToCStringBuf cbuf; + size_t funcIndexStrLen; + const char* funcIndexStr = + Uint32ToCString(&cbuf, funcIndex, &funcIndexStrLen); + MOZ_ASSERT(funcIndexStr); + + return bytes->append(beforeFuncIndex, strlen(beforeFuncIndex)) && + bytes->append(funcIndexStr, funcIndexStrLen) && + bytes->append(afterFuncIndex, strlen(afterFuncIndex)); +} + +bool Metadata::getFuncName(NameContext ctx, uint32_t funcIndex, + UTF8Bytes* name) const { + if (moduleName && moduleName->length != 0) { + if (!AppendName(namePayload->bytes, *moduleName, name)) { + return false; + } + if (!name->append('.')) { + return false; + } + } + + if (funcIndex < funcNames.length() && funcNames[funcIndex].length != 0) { + return AppendName(namePayload->bytes, funcNames[funcIndex], name); + } + + if (ctx == NameContext::BeforeLocation) { + return true; + } + + return AppendFunctionIndexName(funcIndex, name); +} + +bool CodeTier::initialize(const Code& code, const LinkData& linkData, + const Metadata& metadata) { + MOZ_ASSERT(!initialized()); + code_ = &code; + + MOZ_ASSERT(lazyStubs_.readLock()->entryStubsEmpty()); + + // See comments in CodeSegment::initialize() for why this must be last. + if (!segment_->initialize(*this, linkData, metadata, *metadata_)) { + return false; + } + + MOZ_ASSERT(initialized()); + return true; +} + +void CodeTier::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const { + segment_->addSizeOfMisc(mallocSizeOf, code, data); + lazyStubs_.readLock()->addSizeOfMisc(mallocSizeOf, code, data); + *data += metadata_->sizeOfExcludingThis(mallocSizeOf); +} + +const CodeRange* CodeTier::lookupRange(const void* pc) const { + CodeRange::OffsetInCode target((uint8_t*)pc - segment_->base()); + return LookupInSorted(metadata_->codeRanges, target); +} + +const wasm::TryNote* CodeTier::lookupTryNote(const void* pc) const { + size_t target = (uint8_t*)pc - segment_->base(); + const TryNoteVector& tryNotes = metadata_->tryNotes; + + // We find the first hit (there may be multiple) to obtain the innermost + // handler, which is why we cannot binary search here. + for (const auto& tryNote : tryNotes) { + if (tryNote.offsetWithinTryBody(target)) { + return &tryNote; + } + } + + return nullptr; +} + +bool JumpTables::init(CompileMode mode, const ModuleSegment& ms, + const CodeRangeVector& codeRanges) { + static_assert(JSScript::offsetOfJitCodeRaw() == 0, + "wasm fast jit entry is at (void*) jit[funcIndex]"); + + mode_ = mode; + + size_t numFuncs = 0; + for (const CodeRange& cr : codeRanges) { + if (cr.isFunction()) { + numFuncs++; + } + } + + numFuncs_ = numFuncs; + + if (mode_ == CompileMode::Tier1) { + tiering_ = TablePointer(js_pod_calloc(numFuncs)); + if (!tiering_) { + return false; + } + } + + // The number of jit entries is overestimated, but it is simpler when + // filling/looking up the jit entries and safe (worst case we'll crash + // because of a null deref when trying to call the jit entry of an + // unexported function). + jit_ = TablePointer(js_pod_calloc(numFuncs)); + if (!jit_) { + return false; + } + + uint8_t* codeBase = ms.base(); + for (const CodeRange& cr : codeRanges) { + if (cr.isFunction()) { + setTieringEntry(cr.funcIndex(), codeBase + cr.funcTierEntry()); + } else if (cr.isJitEntry()) { + setJitEntry(cr.funcIndex(), codeBase + cr.begin()); + } + } + return true; +} + +Code::Code(UniqueCodeTier tier1, const Metadata& metadata, + JumpTables&& maybeJumpTables) + : tier1_(std::move(tier1)), + metadata_(&metadata), + profilingLabels_(mutexid::WasmCodeProfilingLabels, + CacheableCharsVector()), + jumpTables_(std::move(maybeJumpTables)) {} + +bool Code::initialize(const LinkData& linkData) { + MOZ_ASSERT(!initialized()); + + if (!tier1_->initialize(*this, linkData, *metadata_)) { + return false; + } + + MOZ_ASSERT(initialized()); + return true; +} + +bool Code::setAndBorrowTier2(UniqueCodeTier tier2, const LinkData& linkData, + const CodeTier** borrowedTier) const { + MOZ_RELEASE_ASSERT(!hasTier2()); + MOZ_RELEASE_ASSERT(tier2->tier() == Tier::Optimized && + tier1_->tier() == Tier::Baseline); + + if (!tier2->initialize(*this, linkData, *metadata_)) { + return false; + } + + tier2_ = std::move(tier2); + *borrowedTier = &*tier2_; + + return true; +} + +void Code::commitTier2() const { + MOZ_RELEASE_ASSERT(!hasTier2()); + hasTier2_ = true; + MOZ_ASSERT(hasTier2()); + + // To maintain the invariant that tier2_ is never read without the tier having + // been committed, this checks tier2_ here instead of before setting hasTier2_ + // (as would be natural). See comment in WasmCode.h. + MOZ_RELEASE_ASSERT(tier2_.get()); +} + +uint32_t Code::getFuncIndex(JSFunction* fun) const { + MOZ_ASSERT(fun->isWasm() || fun->isAsmJSNative()); + if (!fun->isWasmWithJitEntry()) { + return fun->wasmFuncIndex(); + } + return jumpTables_.funcIndexFromJitEntry(fun->wasmJitEntry()); +} + +Tiers Code::tiers() const { + if (hasTier2()) { + return Tiers(tier1_->tier(), tier2_->tier()); + } + return Tiers(tier1_->tier()); +} + +bool Code::hasTier(Tier t) const { + if (hasTier2() && tier2_->tier() == t) { + return true; + } + return tier1_->tier() == t; +} + +Tier Code::stableTier() const { return tier1_->tier(); } + +Tier Code::bestTier() const { + if (hasTier2()) { + return tier2_->tier(); + } + return tier1_->tier(); +} + +const CodeTier& Code::codeTier(Tier tier) const { + switch (tier) { + case Tier::Baseline: + if (tier1_->tier() == Tier::Baseline) { + MOZ_ASSERT(tier1_->initialized()); + return *tier1_; + } + MOZ_CRASH("No code segment at this tier"); + case Tier::Optimized: + if (tier1_->tier() == Tier::Optimized) { + MOZ_ASSERT(tier1_->initialized()); + return *tier1_; + } + // It is incorrect to ask for the optimized tier without there being such + // a tier and the tier having been committed. The guard here could + // instead be `if (hasTier2()) ... ` but codeTier(t) should not be called + // in contexts where that test is necessary. + MOZ_RELEASE_ASSERT(hasTier2()); + MOZ_ASSERT(tier2_->initialized()); + return *tier2_; + } + MOZ_CRASH(); +} + +bool Code::containsCodePC(const void* pc) const { + for (Tier t : tiers()) { + const ModuleSegment& ms = segment(t); + if (ms.containsCodePC(pc)) { + return true; + } + } + return false; +} + +struct CallSiteRetAddrOffset { + const CallSiteVector& callSites; + explicit CallSiteRetAddrOffset(const CallSiteVector& callSites) + : callSites(callSites) {} + uint32_t operator[](size_t index) const { + return callSites[index].returnAddressOffset(); + } +}; + +const CallSite* Code::lookupCallSite(void* returnAddress) const { + for (Tier t : tiers()) { + uint32_t target = ((uint8_t*)returnAddress) - segment(t).base(); + size_t lowerBound = 0; + size_t upperBound = metadata(t).callSites.length(); + + size_t match; + if (BinarySearch(CallSiteRetAddrOffset(metadata(t).callSites), lowerBound, + upperBound, target, &match)) { + return &metadata(t).callSites[match]; + } + } + + return nullptr; +} + +const CodeRange* Code::lookupFuncRange(void* pc) const { + for (Tier t : tiers()) { + const CodeRange* result = codeTier(t).lookupRange(pc); + if (result && result->isFunction()) { + return result; + } + } + return nullptr; +} + +const StackMap* Code::lookupStackMap(uint8_t* nextPC) const { + for (Tier t : tiers()) { + const StackMap* result = metadata(t).stackMaps.findMap(nextPC); + if (result) { + return result; + } + } + return nullptr; +} + +const wasm::TryNote* Code::lookupTryNote(void* pc, Tier* tier) const { + for (Tier t : tiers()) { + const TryNote* result = codeTier(t).lookupTryNote(pc); + if (result) { + *tier = t; + return result; + } + } + return nullptr; +} + +struct TrapSitePCOffset { + const TrapSiteVector& trapSites; + explicit TrapSitePCOffset(const TrapSiteVector& trapSites) + : trapSites(trapSites) {} + uint32_t operator[](size_t index) const { return trapSites[index].pcOffset; } +}; + +bool Code::lookupTrap(void* pc, Trap* trapOut, BytecodeOffset* bytecode) const { + for (Tier t : tiers()) { + uint32_t target = ((uint8_t*)pc) - segment(t).base(); + const TrapSiteVectorArray& trapSitesArray = metadata(t).trapSites; + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + const TrapSiteVector& trapSites = trapSitesArray[trap]; + + size_t upperBound = trapSites.length(); + size_t match; + if (BinarySearch(TrapSitePCOffset(trapSites), 0, upperBound, target, + &match)) { + MOZ_ASSERT(segment(t).containsCodePC(pc)); + *trapOut = trap; + *bytecode = trapSites[match].bytecode; + return true; + } + } + } + + return false; +} + +// When enabled, generate profiling labels for every name in funcNames_ that is +// the name of some Function CodeRange. This involves malloc() so do it now +// since, once we start sampling, we'll be in a signal-handing context where we +// cannot malloc. +void Code::ensureProfilingLabels(bool profilingEnabled) const { + auto labels = profilingLabels_.lock(); + + if (!profilingEnabled) { + labels->clear(); + return; + } + + if (!labels->empty()) { + return; + } + + // Any tier will do, we only need tier-invariant data that are incidentally + // stored with the code ranges. + + for (const CodeRange& codeRange : metadata(stableTier()).codeRanges) { + if (!codeRange.isFunction()) { + continue; + } + + Int32ToCStringBuf cbuf; + size_t bytecodeStrLen; + const char* bytecodeStr = + Uint32ToCString(&cbuf, codeRange.funcLineOrBytecode(), &bytecodeStrLen); + MOZ_ASSERT(bytecodeStr); + + UTF8Bytes name; + if (!metadata().getFuncNameStandalone(codeRange.funcIndex(), &name)) { + return; + } + if (!name.append(" (", 2)) { + return; + } + + if (const char* filename = metadata().filename.get()) { + if (!name.append(filename, strlen(filename))) { + return; + } + } else { + if (!name.append('?')) { + return; + } + } + + if (!name.append(':') || !name.append(bytecodeStr, bytecodeStrLen) || + !name.append(")\0", 2)) { + return; + } + + UniqueChars label(name.extractOrCopyRawBuffer()); + if (!label) { + return; + } + + if (codeRange.funcIndex() >= labels->length()) { + if (!labels->resize(codeRange.funcIndex() + 1)) { + return; + } + } + + ((CacheableCharsVector&)labels)[codeRange.funcIndex()] = std::move(label); + } +} + +const char* Code::profilingLabel(uint32_t funcIndex) const { + auto labels = profilingLabels_.lock(); + + if (funcIndex >= labels->length() || + !((CacheableCharsVector&)labels)[funcIndex]) { + return "?"; + } + return ((CacheableCharsVector&)labels)[funcIndex].get(); +} + +void Code::addSizeOfMiscIfNotSeen(MallocSizeOf mallocSizeOf, + Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, size_t* code, + size_t* data) const { + auto p = seenCode->lookupForAdd(this); + if (p) { + return; + } + bool ok = seenCode->add(p, this); + (void)ok; // oh well + + *data += mallocSizeOf(this) + + metadata().sizeOfIncludingThisIfNotSeen(mallocSizeOf, seenMetadata) + + profilingLabels_.lock()->sizeOfExcludingThis(mallocSizeOf) + + jumpTables_.sizeOfMiscExcludingThis(); + + for (auto t : tiers()) { + codeTier(t).addSizeOfMisc(mallocSizeOf, code, data); + } +} + +void Code::disassemble(JSContext* cx, Tier tier, int kindSelection, + PrintCallback printString) const { + const MetadataTier& metadataTier = metadata(tier); + const CodeTier& codeTier = this->codeTier(tier); + const ModuleSegment& segment = codeTier.segment(); + + for (const CodeRange& range : metadataTier.codeRanges) { + if (kindSelection & (1 << range.kind())) { + MOZ_ASSERT(range.begin() < segment.length()); + MOZ_ASSERT(range.end() < segment.length()); + + const char* kind; + char kindbuf[128]; + switch (range.kind()) { + case CodeRange::Function: + kind = "Function"; + break; + case CodeRange::InterpEntry: + kind = "InterpEntry"; + break; + case CodeRange::JitEntry: + kind = "JitEntry"; + break; + case CodeRange::ImportInterpExit: + kind = "ImportInterpExit"; + break; + case CodeRange::ImportJitExit: + kind = "ImportJitExit"; + break; + default: + SprintfLiteral(kindbuf, "CodeRange::Kind(%d)", range.kind()); + kind = kindbuf; + break; + } + const char* separator = + "\n--------------------------------------------------\n"; + // The buffer is quite large in order to accomodate mangled C++ names; + // lengths over 3500 have been observed in the wild. + char buf[4096]; + if (range.hasFuncIndex()) { + const char* funcName = "(unknown)"; + UTF8Bytes namebuf; + if (metadata().getFuncNameStandalone(range.funcIndex(), &namebuf) && + namebuf.append('\0')) { + funcName = namebuf.begin(); + } + SprintfLiteral(buf, "%sKind = %s, index = %d, name = %s:\n", separator, + kind, range.funcIndex(), funcName); + } else { + SprintfLiteral(buf, "%sKind = %s\n", separator, kind); + } + printString(buf); + + uint8_t* theCode = segment.base() + range.begin(); + jit::Disassemble(theCode, range.end() - range.begin(), printString); + } + } +} + +void wasm::PatchDebugSymbolicAccesses(uint8_t* codeBase, MacroAssembler& masm) { +#ifdef WASM_CODEGEN_DEBUG + for (auto& access : masm.symbolicAccesses()) { + switch (access.target) { + case SymbolicAddress::PrintI32: + case SymbolicAddress::PrintPtr: + case SymbolicAddress::PrintF32: + case SymbolicAddress::PrintF64: + case SymbolicAddress::PrintText: + break; + default: + MOZ_CRASH("unexpected symbol in PatchDebugSymbolicAccesses"); + } + ABIFunctionType abiType; + void* target = AddressOf(access.target, &abiType); + uint8_t* patchAt = codeBase + access.patchAt.offset(); + Assembler::PatchDataWithValueCheck(CodeLocationLabel(patchAt), + PatchedImmPtr(target), + PatchedImmPtr((void*)-1)); + } +#else + MOZ_ASSERT(masm.symbolicAccesses().empty()); +#endif +} diff --git a/js/src/wasm/WasmCode.h b/js/src/wasm/WasmCode.h new file mode 100644 index 0000000000..518495199c --- /dev/null +++ b/js/src/wasm/WasmCode.h @@ -0,0 +1,874 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_code_h +#define wasm_code_h + +#include "mozilla/Assertions.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/EnumeratedArray.h" +#include "mozilla/Maybe.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/PodOperations.h" +#include "mozilla/RefPtr.h" +#include "mozilla/UniquePtr.h" + +#include +#include +#include +#include + +#include "jstypes.h" + +#include "gc/Memory.h" +#include "js/AllocPolicy.h" +#include "js/UniquePtr.h" +#include "js/Utility.h" +#include "js/Vector.h" +#include "threading/ExclusiveData.h" +#include "util/Memory.h" +#include "vm/MutexIDs.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCodegenConstants.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmCompileArgs.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmExprType.h" +#include "wasm/WasmGC.h" +#include "wasm/WasmLog.h" +#include "wasm/WasmModuleTypes.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmShareable.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValType.h" + +struct JS_PUBLIC_API JSContext; +class JSFunction; + +namespace js { + +struct AsmJSMetadata; +class ScriptSource; + +namespace jit { +class MacroAssembler; +}; + +namespace wasm { + +struct MetadataTier; +struct Metadata; + +// LinkData contains all the metadata necessary to patch all the locations +// that depend on the absolute address of a ModuleSegment. This happens in a +// "linking" step after compilation and after the module's code is serialized. +// The LinkData is serialized along with the Module but does not (normally, see +// Module::debugLinkData_ comment) persist after (de)serialization, which +// distinguishes it from Metadata, which is stored in the Code object. + +struct LinkDataCacheablePod { + uint32_t trapOffset = 0; + + WASM_CHECK_CACHEABLE_POD(trapOffset); + + LinkDataCacheablePod() = default; +}; + +WASM_DECLARE_CACHEABLE_POD(LinkDataCacheablePod); + +WASM_CHECK_CACHEABLE_POD_PADDING(LinkDataCacheablePod) + +struct LinkData : LinkDataCacheablePod { + explicit LinkData(Tier tier) : tier(tier) {} + + LinkDataCacheablePod& pod() { return *this; } + const LinkDataCacheablePod& pod() const { return *this; } + + struct InternalLink { + uint32_t patchAtOffset; + uint32_t targetOffset; +#ifdef JS_CODELABEL_LINKMODE + uint32_t mode; +#endif + + WASM_CHECK_CACHEABLE_POD(patchAtOffset, targetOffset); +#ifdef JS_CODELABEL_LINKMODE + WASM_CHECK_CACHEABLE_POD(mode) +#endif + }; + using InternalLinkVector = Vector; + + struct SymbolicLinkArray + : EnumeratedArray { + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + }; + + const Tier tier; + InternalLinkVector internalLinks; + SymbolicLinkArray symbolicLinks; + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +WASM_DECLARE_CACHEABLE_POD(LinkData::InternalLink); + +using UniqueLinkData = UniquePtr; + +// Executable code must be deallocated specially. + +struct FreeCode { + uint32_t codeLength; + FreeCode() : codeLength(0) {} + explicit FreeCode(uint32_t codeLength) : codeLength(codeLength) {} + void operator()(uint8_t* codeBytes); +}; + +using UniqueCodeBytes = UniquePtr; + +class Code; +class CodeTier; +class ModuleSegment; +class LazyStubSegment; + +// CodeSegment contains common helpers for determining the base and length of a +// code segment and if a pc belongs to this segment. It is inherited by: +// - ModuleSegment, i.e. the code segment of a Module, generated +// eagerly when a Module is instanciated. +// - LazyStubSegment, i.e. the code segment of entry stubs that are lazily +// generated. + +class CodeSegment { + protected: + enum class Kind { LazyStubs, Module }; + + CodeSegment(UniqueCodeBytes bytes, uint32_t length, Kind kind) + : bytes_(std::move(bytes)), + length_(length), + kind_(kind), + codeTier_(nullptr), + unregisterOnDestroy_(false) {} + + bool initialize(const CodeTier& codeTier); + + private: + const UniqueCodeBytes bytes_; + const uint32_t length_; + const Kind kind_; + const CodeTier* codeTier_; + bool unregisterOnDestroy_; + + public: + bool initialized() const { return !!codeTier_; } + ~CodeSegment(); + + bool isLazyStubs() const { return kind_ == Kind::LazyStubs; } + bool isModule() const { return kind_ == Kind::Module; } + const ModuleSegment* asModule() const { + MOZ_ASSERT(isModule()); + return (ModuleSegment*)this; + } + const LazyStubSegment* asLazyStub() const { + MOZ_ASSERT(isLazyStubs()); + return (LazyStubSegment*)this; + } + + uint8_t* base() const { return bytes_.get(); } + uint32_t length() const { + MOZ_ASSERT(length_ != UINT32_MAX); + return length_; + } + + bool containsCodePC(const void* pc) const { + return pc >= base() && pc < (base() + length_); + } + + const CodeTier& codeTier() const { + MOZ_ASSERT(initialized()); + return *codeTier_; + } + const Code& code() const; + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code) const; +}; + +// A wasm ModuleSegment owns the allocated executable code for a wasm module. + +using UniqueModuleSegment = UniquePtr; + +class ModuleSegment : public CodeSegment { + const Tier tier_; + uint8_t* const trapCode_; + + public: + ModuleSegment(Tier tier, UniqueCodeBytes codeBytes, uint32_t codeLength, + const LinkData& linkData); + + static UniqueModuleSegment create(Tier tier, jit::MacroAssembler& masm, + const LinkData& linkData); + static UniqueModuleSegment create(Tier tier, const Bytes& unlinkedBytes, + const LinkData& linkData); + + bool initialize(const CodeTier& codeTier, const LinkData& linkData, + const Metadata& metadata, const MetadataTier& metadataTier); + + Tier tier() const { return tier_; } + + // Pointers to stubs to which PC is redirected from the signal-handler. + + uint8_t* trapCode() const { return trapCode_; } + + const CodeRange* lookupRange(const void* pc) const; + + void addSizeOfMisc(mozilla::MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const; + + WASM_DECLARE_FRIEND_SERIALIZE(ModuleSegment); +}; + +extern UniqueCodeBytes AllocateCodeBytes(uint32_t codeLength); +extern bool StaticallyLink(const ModuleSegment& ms, const LinkData& linkData); +extern void StaticallyUnlink(uint8_t* base, const LinkData& linkData); + +// A FuncExport represents a single function definition inside a wasm Module +// that has been exported one or more times. A FuncExport represents an +// internal entry point that can be called via function definition index by +// Instance::callExport(). To allow O(log(n)) lookup of a FuncExport by +// function definition index, the FuncExportVector is stored sorted by +// function definition index. + +class FuncExport { + uint32_t typeIndex_; + uint32_t funcIndex_; + uint32_t eagerInterpEntryOffset_; // Machine code offset + bool hasEagerStubs_; + + WASM_CHECK_CACHEABLE_POD(typeIndex_, funcIndex_, eagerInterpEntryOffset_, + hasEagerStubs_); + + public: + FuncExport() = default; + explicit FuncExport(uint32_t typeIndex, uint32_t funcIndex, + bool hasEagerStubs) { + typeIndex_ = typeIndex; + funcIndex_ = funcIndex; + eagerInterpEntryOffset_ = UINT32_MAX; + hasEagerStubs_ = hasEagerStubs; + } + void initEagerInterpEntryOffset(uint32_t entryOffset) { + MOZ_ASSERT(eagerInterpEntryOffset_ == UINT32_MAX); + MOZ_ASSERT(hasEagerStubs()); + eagerInterpEntryOffset_ = entryOffset; + } + + bool hasEagerStubs() const { return hasEagerStubs_; } + uint32_t typeIndex() const { return typeIndex_; } + uint32_t funcIndex() const { return funcIndex_; } + uint32_t eagerInterpEntryOffset() const { + MOZ_ASSERT(eagerInterpEntryOffset_ != UINT32_MAX); + MOZ_ASSERT(hasEagerStubs()); + return eagerInterpEntryOffset_; + } +}; + +WASM_DECLARE_CACHEABLE_POD(FuncExport); + +using FuncExportVector = Vector; + +// An FuncImport contains the runtime metadata needed to implement a call to an +// imported function. Each function import has two call stubs: an optimized path +// into JIT code and a slow path into the generic C++ js::Invoke and these +// offsets of these stubs are stored so that function-import callsites can be +// dynamically patched at runtime. + +class FuncImport { + private: + uint32_t typeIndex_; + uint32_t instanceOffset_; + uint32_t interpExitCodeOffset_; // Machine code offset + uint32_t jitExitCodeOffset_; // Machine code offset + + WASM_CHECK_CACHEABLE_POD(typeIndex_, instanceOffset_, interpExitCodeOffset_, + jitExitCodeOffset_); + + public: + FuncImport() + : typeIndex_(0), + instanceOffset_(0), + interpExitCodeOffset_(0), + jitExitCodeOffset_(0) {} + + FuncImport(uint32_t typeIndex, uint32_t instanceOffset) { + typeIndex_ = typeIndex; + instanceOffset_ = instanceOffset; + interpExitCodeOffset_ = 0; + jitExitCodeOffset_ = 0; + } + + void initInterpExitOffset(uint32_t off) { + MOZ_ASSERT(!interpExitCodeOffset_); + interpExitCodeOffset_ = off; + } + void initJitExitOffset(uint32_t off) { + MOZ_ASSERT(!jitExitCodeOffset_); + jitExitCodeOffset_ = off; + } + + uint32_t typeIndex() const { return typeIndex_; } + uint32_t instanceOffset() const { return instanceOffset_; } + uint32_t interpExitCodeOffset() const { return interpExitCodeOffset_; } + uint32_t jitExitCodeOffset() const { return jitExitCodeOffset_; } +}; + +WASM_DECLARE_CACHEABLE_POD(FuncImport) + +using FuncImportVector = Vector; + +// Metadata holds all the data that is needed to describe compiled wasm code +// at runtime (as opposed to data that is only used to statically link or +// instantiate a module). +// +// Metadata is built incrementally by ModuleGenerator and then shared immutably +// between modules. +// +// The Metadata structure is split into tier-invariant and tier-variant parts; +// the former points to instances of the latter. Additionally, the asm.js +// subsystem subclasses the Metadata, adding more tier-invariant data, some of +// which is serialized. See AsmJS.cpp. + +struct MetadataCacheablePod { + ModuleKind kind; + Maybe memory; + uint32_t instanceDataLength; + Maybe startFuncIndex; + Maybe nameCustomSectionIndex; + bool filenameIsURL; + bool omitsBoundsChecks; + uint32_t typeDefsOffsetStart; + uint32_t tablesOffsetStart; + uint32_t tagsOffsetStart; + uint32_t padding; + + WASM_CHECK_CACHEABLE_POD(kind, memory, instanceDataLength, startFuncIndex, + nameCustomSectionIndex, filenameIsURL, + omitsBoundsChecks, typeDefsOffsetStart, + tablesOffsetStart, tagsOffsetStart) + + explicit MetadataCacheablePod(ModuleKind kind) + : kind(kind), + instanceDataLength(0), + filenameIsURL(false), + omitsBoundsChecks(false), + typeDefsOffsetStart(UINT32_MAX), + tablesOffsetStart(UINT32_MAX), + tagsOffsetStart(UINT32_MAX), + padding(0) {} +}; + +WASM_DECLARE_CACHEABLE_POD(MetadataCacheablePod) + +WASM_CHECK_CACHEABLE_POD_PADDING(MetadataCacheablePod) + +using ModuleHash = uint8_t[8]; + +struct Metadata : public ShareableBase, public MetadataCacheablePod { + SharedTypeContext types; + GlobalDescVector globals; + TableDescVector tables; + TagDescVector tags; + CacheableChars filename; + CacheableChars sourceMapURL; + + // namePayload points at the name section's CustomSection::payload so that + // the Names (which are use payload-relative offsets) can be used + // independently of the Module without duplicating the name section. + SharedBytes namePayload; + Maybe moduleName; + NameVector funcNames; + + // Debug-enabled code is not serialized. + bool debugEnabled; + Uint32Vector debugFuncTypeIndices; + ModuleHash debugHash; + + explicit Metadata(ModuleKind kind = ModuleKind::Wasm) + : MetadataCacheablePod(kind), debugEnabled(false), debugHash() {} + virtual ~Metadata() = default; + + MetadataCacheablePod& pod() { return *this; } + const MetadataCacheablePod& pod() const { return *this; } + + bool usesMemory() const { return memory.isSome(); } + bool usesSharedMemory() const { + return memory.isSome() && memory->isShared(); + } + + const FuncType& getFuncImportType(const FuncImport& funcImport) const { + return types->type(funcImport.typeIndex()).funcType(); + } + const FuncType& getFuncExportType(const FuncExport& funcExport) const { + return types->type(funcExport.typeIndex()).funcType(); + } + + size_t debugNumFuncs() const { return debugFuncTypeIndices.length(); } + const FuncType& debugFuncType(uint32_t funcIndex) const { + MOZ_ASSERT(debugEnabled); + return types->type(debugFuncTypeIndices[funcIndex]).funcType(); + } + + // AsmJSMetadata derives Metadata iff isAsmJS(). Mostly this distinction is + // encapsulated within AsmJS.cpp, but the additional virtual functions allow + // asm.js to override wasm behavior in the handful of cases that can't be + // easily encapsulated by AsmJS.cpp. + + bool isAsmJS() const { return kind == ModuleKind::AsmJS; } + const AsmJSMetadata& asAsmJS() const { + MOZ_ASSERT(isAsmJS()); + return *(const AsmJSMetadata*)this; + } + virtual bool mutedErrors() const { return false; } + virtual const char16_t* displayURL() const { return nullptr; } + virtual ScriptSource* maybeScriptSource() const { return nullptr; } + + // The Developer-Facing Display Conventions section of the WebAssembly Web + // API spec defines two cases for displaying a wasm function name: + // 1. the function name stands alone + // 2. the function name precedes the location + + enum NameContext { Standalone, BeforeLocation }; + + virtual bool getFuncName(NameContext ctx, uint32_t funcIndex, + UTF8Bytes* name) const; + + bool getFuncNameStandalone(uint32_t funcIndex, UTF8Bytes* name) const { + return getFuncName(NameContext::Standalone, funcIndex, name); + } + bool getFuncNameBeforeLocation(uint32_t funcIndex, UTF8Bytes* name) const { + return getFuncName(NameContext::BeforeLocation, funcIndex, name); + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(Metadata); +}; + +using MutableMetadata = RefPtr; +using SharedMetadata = RefPtr; + +struct MetadataTier { + explicit MetadataTier(Tier tier = Tier::Serialized) + : tier(tier), debugTrapOffset(0) {} + + const Tier tier; + + Uint32Vector funcToCodeRange; + CodeRangeVector codeRanges; + CallSiteVector callSites; + TrapSiteVectorArray trapSites; + FuncImportVector funcImports; + FuncExportVector funcExports; + StackMaps stackMaps; + TryNoteVector tryNotes; + + // Debug information, not serialized. + uint32_t debugTrapOffset; + + FuncExport& lookupFuncExport(uint32_t funcIndex, + size_t* funcExportIndex = nullptr); + const FuncExport& lookupFuncExport(uint32_t funcIndex, + size_t* funcExportIndex = nullptr) const; + + const CodeRange& codeRange(const FuncExport& funcExport) const { + return codeRanges[funcToCodeRange[funcExport.funcIndex()]]; + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using UniqueMetadataTier = UniquePtr; + +// LazyStubSegment is a code segment lazily generated for function entry stubs +// (both interpreter and jit ones). +// +// Because a stub is usually small (a few KiB) and an executable code segment +// isn't (64KiB), a given stub segment can contain entry stubs of many +// functions. + +using UniqueLazyStubSegment = UniquePtr; +using LazyStubSegmentVector = + Vector; + +class LazyStubSegment : public CodeSegment { + CodeRangeVector codeRanges_; + size_t usedBytes_; + + public: + LazyStubSegment(UniqueCodeBytes bytes, size_t length) + : CodeSegment(std::move(bytes), length, CodeSegment::Kind::LazyStubs), + usedBytes_(0) {} + + static UniqueLazyStubSegment create(const CodeTier& codeTier, + size_t codeLength); + + static size_t AlignBytesNeeded(size_t bytes) { + return AlignBytes(bytes, gc::SystemPageSize()); + } + + bool hasSpace(size_t bytes) const; + [[nodiscard]] bool addStubs(const Metadata& metadata, size_t codeLength, + const Uint32Vector& funcExportIndices, + const FuncExportVector& funcExports, + const CodeRangeVector& codeRanges, + uint8_t** codePtr, + size_t* indexFirstInsertedCodeRange); + + const CodeRangeVector& codeRanges() const { return codeRanges_; } + [[nodiscard]] const CodeRange* lookupRange(const void* pc) const; + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const; +}; + +// LazyFuncExport helps to efficiently lookup a CodeRange from a given function +// index. It is inserted in a vector sorted by function index, to perform +// binary search on it later. + +struct LazyFuncExport { + size_t funcIndex; + size_t lazyStubSegmentIndex; + size_t funcCodeRangeIndex; + LazyFuncExport(size_t funcIndex, size_t lazyStubSegmentIndex, + size_t funcCodeRangeIndex) + : funcIndex(funcIndex), + lazyStubSegmentIndex(lazyStubSegmentIndex), + funcCodeRangeIndex(funcCodeRangeIndex) {} +}; + +using LazyFuncExportVector = Vector; + +// LazyStubTier contains all the necessary information for lazy function entry +// stubs that are generated at runtime. None of its data are ever serialized. +// +// It must be protected by a lock, because the main thread can both read and +// write lazy stubs at any time while a background thread can regenerate lazy +// stubs for tier2 at any time. + +class LazyStubTier { + LazyStubSegmentVector stubSegments_; + LazyFuncExportVector exports_; + size_t lastStubSegmentIndex_; + + [[nodiscard]] bool createManyEntryStubs(const Uint32Vector& funcExportIndices, + const Metadata& metadata, + const CodeTier& codeTier, + size_t* stubSegmentIndex); + + public: + LazyStubTier() : lastStubSegmentIndex_(0) {} + + // Creates one lazy stub for the exported function, for which the jit entry + // will be set to the lazily-generated one. + [[nodiscard]] bool createOneEntryStub(uint32_t funcExportIndex, + const Metadata& metadata, + const CodeTier& codeTier); + + bool entryStubsEmpty() const { return stubSegments_.empty(); } + bool hasEntryStub(uint32_t funcIndex) const; + + // Returns a pointer to the raw interpreter entry of a given function for + // which stubs have been lazily generated. + [[nodiscard]] void* lookupInterpEntry(uint32_t funcIndex) const; + + // Create one lazy stub for all the functions in funcExportIndices, putting + // them in a single stub. Jit entries won't be used until + // setJitEntries() is actually called, after the Code owner has committed + // tier2. + [[nodiscard]] bool createTier2(const Uint32Vector& funcExportIndices, + const Metadata& metadata, + const CodeTier& codeTier, + Maybe* stubSegmentIndex); + void setJitEntries(const Maybe& stubSegmentIndex, const Code& code); + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const; +}; + +// CodeTier contains all the data related to a given compilation tier. It is +// built during module generation and then immutably stored in a Code. + +using UniqueCodeTier = UniquePtr; +using UniqueConstCodeTier = UniquePtr; + +class CodeTier { + const Code* code_; + + // Serialized information. + const UniqueMetadataTier metadata_; + const UniqueModuleSegment segment_; + + // Lazy stubs, not serialized. + RWExclusiveData lazyStubs_; + + static const MutexId& mutexForTier(Tier tier) { + if (tier == Tier::Baseline) { + return mutexid::WasmLazyStubsTier1; + } + MOZ_ASSERT(tier == Tier::Optimized); + return mutexid::WasmLazyStubsTier2; + } + + public: + CodeTier(UniqueMetadataTier metadata, UniqueModuleSegment segment) + : code_(nullptr), + metadata_(std::move(metadata)), + segment_(std::move(segment)), + lazyStubs_(mutexForTier(segment_->tier())) {} + + bool initialized() const { return !!code_ && segment_->initialized(); } + bool initialize(const Code& code, const LinkData& linkData, + const Metadata& metadata); + + Tier tier() const { return segment_->tier(); } + const RWExclusiveData& lazyStubs() const { return lazyStubs_; } + const MetadataTier& metadata() const { return *metadata_.get(); } + const ModuleSegment& segment() const { return *segment_.get(); } + const Code& code() const { + MOZ_ASSERT(initialized()); + return *code_; + } + + const CodeRange* lookupRange(const void* pc) const; + const TryNote* lookupTryNote(const void* pc) const; + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* code, + size_t* data) const; + + WASM_DECLARE_FRIEND_SERIALIZE_ARGS(CodeTier, const wasm::LinkData& data); +}; + +// Jump tables that implement function tiering and fast js-to-wasm calls. +// +// There is one JumpTable object per Code object, holding two jump tables: the +// tiering jump table and the jit-entry jump table. The JumpTable is not +// serialized with its Code, but is a run-time entity only. At run-time it is +// shared across threads with its owning Code (and the Module that owns the +// Code). Values in the JumpTable /must/ /always/ be JSContext-agnostic and +// Instance-agnostic, because of this sharing. +// +// Both jump tables have a number of entries equal to the number of functions in +// their Module, including imports. In the tiering table, the elements +// corresponding to the Module's imported functions are unused; in the jit-entry +// table, the elements corresponding to the Module's non-exported functions are +// unused. (Functions can be exported explicitly via the exports section or +// implicitly via a mention of their indices outside function bodies.) See +// comments at JumpTables::init() and WasmInstanceObject::getExportedFunction(). +// The entries are void*. Unused entries are null. +// +// The tiering jump table. +// +// This table holds code pointers that are used by baseline functions to enter +// optimized code. See the large comment block in WasmCompile.cpp for +// information about how tiering works. +// +// The jit-entry jump table. +// +// The jit-entry jump table entry for a function holds a stub that allows Jitted +// JS code to call wasm using the JS JIT ABI. See large comment block at +// WasmInstanceObject::getExportedFunction() for more about exported functions +// and stubs and the lifecycle of the entries in the jit-entry table - there are +// complex invariants. + +class JumpTables { + using TablePointer = mozilla::UniquePtr; + + CompileMode mode_; + TablePointer tiering_; + TablePointer jit_; + size_t numFuncs_; + + static_assert( + JumpTableJitEntryOffset == 0, + "Each jit entry in table must have compatible layout with BaseScript and" + "SelfHostedLazyScript"); + + public: + bool init(CompileMode mode, const ModuleSegment& ms, + const CodeRangeVector& codeRanges); + + void setJitEntry(size_t i, void* target) const { + // Make sure that write is atomic; see comment in wasm::Module::finishTier2 + // to that effect. + MOZ_ASSERT(i < numFuncs_); + jit_.get()[i] = target; + } + void setJitEntryIfNull(size_t i, void* target) const { + // Make sure that compare-and-write is atomic; see comment in + // wasm::Module::finishTier2 to that effect. + MOZ_ASSERT(i < numFuncs_); + void* expected = nullptr; + (void)__atomic_compare_exchange_n(&jit_.get()[i], &expected, target, + /*weak=*/false, __ATOMIC_RELAXED, + __ATOMIC_RELAXED); + } + void** getAddressOfJitEntry(size_t i) const { + MOZ_ASSERT(i < numFuncs_); + MOZ_ASSERT(jit_.get()[i]); + return &jit_.get()[i]; + } + size_t funcIndexFromJitEntry(void** target) const { + MOZ_ASSERT(target >= &jit_.get()[0]); + MOZ_ASSERT(target <= &(jit_.get()[numFuncs_ - 1])); + return (intptr_t*)target - (intptr_t*)&jit_.get()[0]; + } + + void setTieringEntry(size_t i, void* target) const { + MOZ_ASSERT(i < numFuncs_); + // See comment in wasm::Module::finishTier2. + if (mode_ == CompileMode::Tier1) { + tiering_.get()[i] = target; + } + } + void** tiering() const { return tiering_.get(); } + + size_t sizeOfMiscExcludingThis() const { + // 2 words per function for the jit entry table, plus maybe 1 per + // function if we're tiering. + return sizeof(void*) * (2 + (tiering_ ? 1 : 0)) * numFuncs_; + } +}; + +// Code objects own executable code and the metadata that describe it. A single +// Code object is normally shared between a module and all its instances. +// +// profilingLabels_ is lazily initialized, but behind a lock. + +using SharedCode = RefPtr; +using MutableCode = RefPtr; + +class Code : public ShareableBase { + UniqueCodeTier tier1_; + + // [SMDOC] Tier-2 data + // + // hasTier2_ and tier2_ implement a three-state protocol for broadcasting + // tier-2 data; this also amounts to a single-writer/multiple-reader setup. + // + // Initially hasTier2_ is false and tier2_ is null. + // + // While hasTier2_ is false, *no* thread may read tier2_, but one thread may + // make tier2_ non-null (this will be the tier-2 compiler thread). That same + // thread must then later set hasTier2_ to true to broadcast the tier2_ value + // and its availability. Note that the writing thread may not itself read + // tier2_ before setting hasTier2_, in order to simplify reasoning about + // global invariants. + // + // Once hasTier2_ is true, *no* thread may write tier2_ and *no* thread may + // read tier2_ without having observed hasTier2_ as true first. Once + // hasTier2_ is true, it stays true. + mutable UniqueConstCodeTier tier2_; + mutable Atomic hasTier2_; + + SharedMetadata metadata_; + ExclusiveData profilingLabels_; + JumpTables jumpTables_; + + public: + Code(UniqueCodeTier tier1, const Metadata& metadata, + JumpTables&& maybeJumpTables); + bool initialized() const { return tier1_->initialized(); } + + bool initialize(const LinkData& linkData); + + void setTieringEntry(size_t i, void* target) const { + jumpTables_.setTieringEntry(i, target); + } + void** tieringJumpTable() const { return jumpTables_.tiering(); } + + void setJitEntry(size_t i, void* target) const { + jumpTables_.setJitEntry(i, target); + } + void setJitEntryIfNull(size_t i, void* target) const { + jumpTables_.setJitEntryIfNull(i, target); + } + void** getAddressOfJitEntry(size_t i) const { + return jumpTables_.getAddressOfJitEntry(i); + } + uint32_t getFuncIndex(JSFunction* fun) const; + + // Install the tier2 code without committing it. To maintain the invariant + // that tier2_ is never accessed without the tier having been committed, this + // returns a pointer to the installed tier that the caller can use for + // subsequent operations. + bool setAndBorrowTier2(UniqueCodeTier tier2, const LinkData& linkData, + const CodeTier** borrowedTier) const; + void commitTier2() const; + + bool hasTier2() const { return hasTier2_; } + Tiers tiers() const; + bool hasTier(Tier t) const; + + Tier stableTier() const; // This is stable during a run + Tier bestTier() + const; // This may transition from Baseline -> Ion at any time + + const CodeTier& codeTier(Tier tier) const; + const Metadata& metadata() const { return *metadata_; } + + const ModuleSegment& segment(Tier iter) const { + return codeTier(iter).segment(); + } + const MetadataTier& metadata(Tier iter) const { + return codeTier(iter).metadata(); + } + + // Metadata lookup functions: + + const CallSite* lookupCallSite(void* returnAddress) const; + const CodeRange* lookupFuncRange(void* pc) const; + const StackMap* lookupStackMap(uint8_t* nextPC) const; + const TryNote* lookupTryNote(void* pc, Tier* tier) const; + bool containsCodePC(const void* pc) const; + bool lookupTrap(void* pc, Trap* trap, BytecodeOffset* bytecode) const; + + // To save memory, profilingLabels_ are generated lazily when profiling mode + // is enabled. + + void ensureProfilingLabels(bool profilingEnabled) const; + const char* profilingLabel(uint32_t funcIndex) const; + + // Wasm disassembly support + + void disassemble(JSContext* cx, Tier tier, int kindSelection, + PrintCallback printString) const; + + // about:memory reporting: + + void addSizeOfMiscIfNotSeen(MallocSizeOf mallocSizeOf, + Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, size_t* code, + size_t* data) const; + + WASM_DECLARE_FRIEND_SERIALIZE_ARGS(SharedCode, const wasm::LinkData& data); +}; + +void PatchDebugSymbolicAccesses(uint8_t* codeBase, jit::MacroAssembler& masm); + +} // namespace wasm +} // namespace js + +#endif // wasm_code_h diff --git a/js/src/wasm/WasmCodegenConstants.h b/js/src/wasm/WasmCodegenConstants.h new file mode 100644 index 0000000000..8577c646ff --- /dev/null +++ b/js/src/wasm/WasmCodegenConstants.h @@ -0,0 +1,76 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_codegen_constants_h +#define wasm_codegen_constants_h + +#include + +namespace js { +namespace wasm { + +static const unsigned MaxArgsForJitInlineCall = 8; +static const unsigned MaxResultsForJitEntry = 1; +static const unsigned MaxResultsForJitExit = 1; +static const unsigned MaxResultsForJitInlineCall = MaxResultsForJitEntry; +// The maximum number of results of a function call or block that may be +// returned in registers. +static const unsigned MaxRegisterResults = 1; + +// A magic value of the InstanceReg to indicate after a return to the entry +// stub that an exception has been caught and that we should throw. + +static const unsigned FailInstanceReg = 0xbad; + +// The following thresholds were derived from a microbenchmark. If we begin to +// ship this optimization for more platforms, we will need to extend this list. + +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64) +static const uint32_t MaxInlineMemoryCopyLength = 64; +static const uint32_t MaxInlineMemoryFillLength = 64; +#elif defined(JS_CODEGEN_X86) +static const uint32_t MaxInlineMemoryCopyLength = 32; +static const uint32_t MaxInlineMemoryFillLength = 32; +#else +static const uint32_t MaxInlineMemoryCopyLength = 0; +static const uint32_t MaxInlineMemoryFillLength = 0; +#endif + +// The size we round all super type vectors to. All accesses below this length +// can avoid bounds checks. The value of 8 was chosen after a bit of profiling +// with the Dart Barista benchmark. +// +// Keep jit-tests/tests/wasm/gc/casting.js in sync with this constant. +static const uint32_t MinSuperTypeVectorLength = 8; + +// An exported wasm function may have a 'jit entry' stub attached that can be +// called using the JS JIT ABI. This relies on the pointer we store in the +// `NativeJitInfoOrInterpretedScriptSlot` slot of JSFunction to have a +// compatible representation with BaseScript/SelfHostedLazyScript so that +// `masm.loadJitCodeRaw` works. +// +// We store jit entry pointers in an array (see wasm::JumpTable) and store the +// pointer to a function's jit entry in the JSFunction slot. We rely on the +// below offset of each entry in the jump table to be compatible with +// BaseScript/SelfHostedLazyScript. +static const uint32_t JumpTableJitEntryOffset = 0; + +} // namespace wasm +} // namespace js + +#endif // wasm_codegen_constants_h diff --git a/js/src/wasm/WasmCodegenTypes.cpp b/js/src/wasm/WasmCodegenTypes.cpp new file mode 100644 index 0000000000..6460fd3322 --- /dev/null +++ b/js/src/wasm/WasmCodegenTypes.cpp @@ -0,0 +1,240 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmCodegenTypes.h" + +#include "wasm/WasmExprType.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValidate.h" +#include "wasm/WasmValue.h" + +using mozilla::MakeEnumeratedRange; +using mozilla::PodZero; + +using namespace js; +using namespace js::wasm; + +ArgTypeVector::ArgTypeVector(const FuncType& funcType) + : args_(funcType.args()), + hasStackResults_(ABIResultIter::HasStackResults( + ResultType::Vector(funcType.results()))) {} + +bool TrapSiteVectorArray::empty() const { + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + if (!(*this)[trap].empty()) { + return false; + } + } + + return true; +} + +void TrapSiteVectorArray::clear() { + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + (*this)[trap].clear(); + } +} + +void TrapSiteVectorArray::swap(TrapSiteVectorArray& rhs) { + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + (*this)[trap].swap(rhs[trap]); + } +} + +void TrapSiteVectorArray::shrinkStorageToFit() { + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + (*this)[trap].shrinkStorageToFit(); + } +} + +size_t TrapSiteVectorArray::sizeOfExcludingThis( + MallocSizeOf mallocSizeOf) const { + size_t ret = 0; + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + ret += (*this)[trap].sizeOfExcludingThis(mallocSizeOf); + } + return ret; +} + +CodeRange::CodeRange(Kind kind, Offsets offsets) + : begin_(offsets.begin), ret_(0), end_(offsets.end), kind_(kind) { + MOZ_ASSERT(begin_ <= end_); + PodZero(&u); +#ifdef DEBUG + switch (kind_) { + case FarJumpIsland: + case TrapExit: + case Throw: + break; + default: + MOZ_CRASH("should use more specific constructor"); + } +#endif +} + +CodeRange::CodeRange(Kind kind, uint32_t funcIndex, Offsets offsets) + : begin_(offsets.begin), ret_(0), end_(offsets.end), kind_(kind) { + u.funcIndex_ = funcIndex; + u.func.lineOrBytecode_ = 0; + u.func.beginToUncheckedCallEntry_ = 0; + u.func.beginToTierEntry_ = 0; + MOZ_ASSERT(isEntry()); + MOZ_ASSERT(begin_ <= end_); +} + +CodeRange::CodeRange(Kind kind, CallableOffsets offsets) + : begin_(offsets.begin), ret_(offsets.ret), end_(offsets.end), kind_(kind) { + MOZ_ASSERT(begin_ < ret_); + MOZ_ASSERT(ret_ < end_); + PodZero(&u); +#ifdef DEBUG + switch (kind_) { + case DebugTrap: + case BuiltinThunk: + break; + default: + MOZ_CRASH("should use more specific constructor"); + } +#endif +} + +CodeRange::CodeRange(Kind kind, uint32_t funcIndex, CallableOffsets offsets) + : begin_(offsets.begin), ret_(offsets.ret), end_(offsets.end), kind_(kind) { + MOZ_ASSERT(isImportExit() || isJitEntry()); + MOZ_ASSERT(begin_ < ret_); + MOZ_ASSERT(ret_ < end_); + u.funcIndex_ = funcIndex; + u.func.lineOrBytecode_ = 0; + u.func.beginToUncheckedCallEntry_ = 0; + u.func.beginToTierEntry_ = 0; +} + +CodeRange::CodeRange(uint32_t funcIndex, uint32_t funcLineOrBytecode, + FuncOffsets offsets) + : begin_(offsets.begin), + ret_(offsets.ret), + end_(offsets.end), + kind_(Function) { + MOZ_ASSERT(begin_ < ret_); + MOZ_ASSERT(ret_ < end_); + MOZ_ASSERT(offsets.uncheckedCallEntry - begin_ <= UINT16_MAX); + MOZ_ASSERT(offsets.tierEntry - begin_ <= UINT16_MAX); + u.funcIndex_ = funcIndex; + u.func.lineOrBytecode_ = funcLineOrBytecode; + u.func.beginToUncheckedCallEntry_ = offsets.uncheckedCallEntry - begin_; + u.func.beginToTierEntry_ = offsets.tierEntry - begin_; +} + +const CodeRange* wasm::LookupInSorted(const CodeRangeVector& codeRanges, + CodeRange::OffsetInCode target) { + size_t lowerBound = 0; + size_t upperBound = codeRanges.length(); + + size_t match; + if (!BinarySearch(codeRanges, lowerBound, upperBound, target, &match)) { + return nullptr; + } + + return &codeRanges[match]; +} + +CallIndirectId CallIndirectId::forAsmJSFunc() { + return CallIndirectId(CallIndirectIdKind::AsmJS, 0); +} + +CallIndirectId CallIndirectId::forFunc(const ModuleEnvironment& moduleEnv, + uint32_t funcIndex) { + // asm.js tables are homogenous and don't require a signature check + if (moduleEnv.isAsmJS()) { + return CallIndirectId::forAsmJSFunc(); + } + + FuncDesc func = moduleEnv.funcs[funcIndex]; + if (!func.canRefFunc()) { + return CallIndirectId(); + } + return CallIndirectId::forFuncType(moduleEnv, + moduleEnv.funcs[funcIndex].typeIndex); +} + +CallIndirectId CallIndirectId::forFuncType(const ModuleEnvironment& moduleEnv, + uint32_t funcTypeIndex) { + // asm.js tables are homogenous and don't require a signature check + if (moduleEnv.isAsmJS()) { + return CallIndirectId::forAsmJSFunc(); + } + + const FuncType& funcType = moduleEnv.types->type(funcTypeIndex).funcType(); + if (funcType.hasImmediateTypeId()) { + return CallIndirectId(CallIndirectIdKind::Immediate, + funcType.immediateTypeId()); + } + return CallIndirectId(CallIndirectIdKind::Global, + moduleEnv.offsetOfTypeDef(funcTypeIndex)); +} + +CalleeDesc CalleeDesc::function(uint32_t funcIndex) { + CalleeDesc c; + c.which_ = Func; + c.u.funcIndex_ = funcIndex; + return c; +} +CalleeDesc CalleeDesc::import(uint32_t instanceDataOffset) { + CalleeDesc c; + c.which_ = Import; + c.u.import.instanceDataOffset_ = instanceDataOffset; + return c; +} +CalleeDesc CalleeDesc::wasmTable(const ModuleEnvironment& moduleEnv, + const TableDesc& desc, uint32_t tableIndex, + CallIndirectId callIndirectId) { + CalleeDesc c; + c.which_ = WasmTable; + c.u.table.instanceDataOffset_ = + moduleEnv.offsetOfTableInstanceData(tableIndex); + c.u.table.minLength_ = desc.initialLength; + c.u.table.maxLength_ = desc.maximumLength; + c.u.table.callIndirectId_ = callIndirectId; + return c; +} +CalleeDesc CalleeDesc::asmJSTable(const ModuleEnvironment& moduleEnv, + uint32_t tableIndex) { + CalleeDesc c; + c.which_ = AsmJSTable; + c.u.table.instanceDataOffset_ = + moduleEnv.offsetOfTableInstanceData(tableIndex); + return c; +} +CalleeDesc CalleeDesc::builtin(SymbolicAddress callee) { + CalleeDesc c; + c.which_ = Builtin; + c.u.builtin_ = callee; + return c; +} +CalleeDesc CalleeDesc::builtinInstanceMethod(SymbolicAddress callee) { + CalleeDesc c; + c.which_ = BuiltinInstanceMethod; + c.u.builtin_ = callee; + return c; +} +CalleeDesc CalleeDesc::wasmFuncRef() { + CalleeDesc c; + c.which_ = FuncRef; + return c; +} diff --git a/js/src/wasm/WasmCodegenTypes.h b/js/src/wasm/WasmCodegenTypes.h new file mode 100644 index 0000000000..162298d4a8 --- /dev/null +++ b/js/src/wasm/WasmCodegenTypes.h @@ -0,0 +1,765 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_codegen_types_h +#define wasm_codegen_types_h + +#include "mozilla/EnumeratedArray.h" +#include "mozilla/PodOperations.h" + +#include + +#include "jit/IonTypes.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCodegenConstants.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmInstanceData.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmUtility.h" + +namespace js { + +namespace jit { +template +class ABIArgIterBase; +} // namespace jit + +namespace wasm { + +using mozilla::EnumeratedArray; + +struct ModuleEnvironment; +struct TableDesc; +struct V128; + +// ArgTypeVector type. +// +// Functions usually receive one ABI argument per WebAssembly argument. However +// if a function has multiple results and some of those results go to the stack, +// then it additionally receives a synthetic ABI argument holding a pointer to +// the stack result area. +// +// Given the presence of synthetic arguments, sometimes we need a name for +// non-synthetic arguments. We call those "natural" arguments. + +enum class StackResults { HasStackResults, NoStackResults }; + +class ArgTypeVector { + const ValTypeVector& args_; + bool hasStackResults_; + + // To allow ABIArgIterBase, we define a private + // length() method. To prevent accidental errors, other users need to be + // explicit and call lengthWithStackResults() or + // lengthWithoutStackResults(). + size_t length() const { return args_.length() + size_t(hasStackResults_); } + template + friend class jit::ABIArgIterBase; + + public: + ArgTypeVector(const ValTypeVector& args, StackResults stackResults) + : args_(args), + hasStackResults_(stackResults == StackResults::HasStackResults) {} + explicit ArgTypeVector(const FuncType& funcType); + + bool hasSyntheticStackResultPointerArg() const { return hasStackResults_; } + StackResults stackResults() const { + return hasSyntheticStackResultPointerArg() ? StackResults::HasStackResults + : StackResults::NoStackResults; + } + size_t lengthWithoutStackResults() const { return args_.length(); } + bool isSyntheticStackResultPointerArg(size_t idx) const { + // The pointer to stack results area, if present, is a synthetic argument + // tacked on at the end. + MOZ_ASSERT(idx < lengthWithStackResults()); + return idx == args_.length(); + } + bool isNaturalArg(size_t idx) const { + return !isSyntheticStackResultPointerArg(idx); + } + size_t naturalIndex(size_t idx) const { + MOZ_ASSERT(isNaturalArg(idx)); + // Because the synthetic argument, if present, is tacked on the end, an + // argument index that isn't synthetic is natural. + return idx; + } + + size_t lengthWithStackResults() const { return length(); } + jit::MIRType operator[](size_t i) const { + MOZ_ASSERT(i < lengthWithStackResults()); + if (isSyntheticStackResultPointerArg(i)) { + return jit::MIRType::StackResults; + } + return args_[naturalIndex(i)].toMIRType(); + } +}; + +// A wrapper around the bytecode offset of a wasm instruction within a whole +// module, used for trap offsets or call offsets. These offsets should refer to +// the first byte of the instruction that triggered the trap / did the call and +// should ultimately derive from OpIter::bytecodeOffset. + +class BytecodeOffset { + static const uint32_t INVALID = -1; + uint32_t offset_; + + WASM_CHECK_CACHEABLE_POD(offset_); + + public: + BytecodeOffset() : offset_(INVALID) {} + explicit BytecodeOffset(uint32_t offset) : offset_(offset) {} + + bool isValid() const { return offset_ != INVALID; } + uint32_t offset() const { + MOZ_ASSERT(isValid()); + return offset_; + } +}; + +WASM_DECLARE_CACHEABLE_POD(BytecodeOffset); + +// A TrapSite (in the TrapSiteVector for a given Trap code) represents a wasm +// instruction at a given bytecode offset that can fault at the given pc offset. +// When such a fault occurs, a signal/exception handler looks up the TrapSite to +// confirm the fault is intended/safe and redirects pc to the trap stub. + +struct TrapSite { + uint32_t pcOffset; + BytecodeOffset bytecode; + + WASM_CHECK_CACHEABLE_POD(pcOffset, bytecode); + + TrapSite() : pcOffset(-1), bytecode() {} + TrapSite(uint32_t pcOffset, BytecodeOffset bytecode) + : pcOffset(pcOffset), bytecode(bytecode) {} + + void offsetBy(uint32_t offset) { pcOffset += offset; } +}; + +WASM_DECLARE_CACHEABLE_POD(TrapSite); +WASM_DECLARE_POD_VECTOR(TrapSite, TrapSiteVector) + +struct TrapSiteVectorArray + : EnumeratedArray { + bool empty() const; + void clear(); + void swap(TrapSiteVectorArray& rhs); + void shrinkStorageToFit(); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +// On trap, the bytecode offset to be reported in callstacks is saved. + +struct TrapData { + // The resumePC indicates where, if the trap doesn't throw, the trap stub + // should jump to after restoring all register state. + void* resumePC; + + // The unwoundPC is the PC after adjustment by wasm::StartUnwinding(), which + // basically unwinds partially-construted wasm::Frames when pc is in the + // prologue/epilogue. Stack traces during a trap should use this PC since + // it corresponds to the JitActivation::wasmExitFP. + void* unwoundPC; + + Trap trap; + uint32_t bytecodeOffset; +}; + +// The (,Callable,Func)Offsets classes are used to record the offsets of +// different key points in a CodeRange during compilation. + +struct Offsets { + explicit Offsets(uint32_t begin = 0, uint32_t end = 0) + : begin(begin), end(end) {} + + // These define a [begin, end) contiguous range of instructions compiled + // into a CodeRange. + uint32_t begin; + uint32_t end; + + WASM_CHECK_CACHEABLE_POD(begin, end); +}; + +WASM_DECLARE_CACHEABLE_POD(Offsets); + +struct CallableOffsets : Offsets { + MOZ_IMPLICIT CallableOffsets(uint32_t ret = 0) : Offsets(), ret(ret) {} + + // The offset of the return instruction precedes 'end' by a variable number + // of instructions due to out-of-line codegen. + uint32_t ret; + + WASM_CHECK_CACHEABLE_POD_WITH_PARENT(Offsets, ret); +}; + +WASM_DECLARE_CACHEABLE_POD(CallableOffsets); + +struct FuncOffsets : CallableOffsets { + MOZ_IMPLICIT FuncOffsets() + : CallableOffsets(), uncheckedCallEntry(0), tierEntry(0) {} + + // Function CodeRanges have a checked call entry which takes an extra + // signature argument which is checked against the callee's signature before + // falling through to the normal prologue. The checked call entry is thus at + // the beginning of the CodeRange and the unchecked call entry is at some + // offset after the checked call entry. + // + // Note that there won't always be a checked call entry because not all + // functions require them. See GenerateFunctionPrologue. + uint32_t uncheckedCallEntry; + + // The tierEntry is the point within a function to which the patching code + // within a Tier-1 function jumps. It could be the instruction following + // the jump in the Tier-1 function, or the point following the standard + // prologue within a Tier-2 function. + uint32_t tierEntry; + + WASM_CHECK_CACHEABLE_POD_WITH_PARENT(CallableOffsets, uncheckedCallEntry, + tierEntry); +}; + +WASM_DECLARE_CACHEABLE_POD(FuncOffsets); + +using FuncOffsetsVector = Vector; + +// A CodeRange describes a single contiguous range of code within a wasm +// module's code segment. A CodeRange describes what the code does and, for +// function bodies, the name and source coordinates of the function. + +class CodeRange { + public: + enum Kind { + Function, // function definition + InterpEntry, // calls into wasm from C++ + JitEntry, // calls into wasm from jit code + ImportInterpExit, // slow-path calling from wasm into C++ interp + ImportJitExit, // fast-path calling from wasm into jit code + BuiltinThunk, // fast-path calling from wasm into a C++ native + TrapExit, // calls C++ to report and jumps to throw stub + DebugTrap, // calls C++ to handle debug event + FarJumpIsland, // inserted to connect otherwise out-of-range insns + Throw // special stack-unwinding stub jumped to by other stubs + }; + + private: + // All fields are treated as cacheable POD: + uint32_t begin_; + uint32_t ret_; + uint32_t end_; + union { + struct { + uint32_t funcIndex_; + union { + struct { + uint32_t lineOrBytecode_; + uint16_t beginToUncheckedCallEntry_; + uint16_t beginToTierEntry_; + } func; + }; + }; + Trap trap_; + } u; + Kind kind_ : 8; + + WASM_CHECK_CACHEABLE_POD(begin_, ret_, end_, u.funcIndex_, + u.func.lineOrBytecode_, + u.func.beginToUncheckedCallEntry_, + u.func.beginToTierEntry_, u.trap_, kind_); + + public: + CodeRange() = default; + CodeRange(Kind kind, Offsets offsets); + CodeRange(Kind kind, uint32_t funcIndex, Offsets offsets); + CodeRange(Kind kind, CallableOffsets offsets); + CodeRange(Kind kind, uint32_t funcIndex, CallableOffsets); + CodeRange(uint32_t funcIndex, uint32_t lineOrBytecode, FuncOffsets offsets); + + void offsetBy(uint32_t offset) { + begin_ += offset; + end_ += offset; + if (hasReturn()) { + ret_ += offset; + } + } + + // All CodeRanges have a begin and end. + + uint32_t begin() const { return begin_; } + uint32_t end() const { return end_; } + + // Other fields are only available for certain CodeRange::Kinds. + + Kind kind() const { return kind_; } + + bool isFunction() const { return kind() == Function; } + bool isImportExit() const { + return kind() == ImportJitExit || kind() == ImportInterpExit || + kind() == BuiltinThunk; + } + bool isImportInterpExit() const { return kind() == ImportInterpExit; } + bool isImportJitExit() const { return kind() == ImportJitExit; } + bool isTrapExit() const { return kind() == TrapExit; } + bool isDebugTrap() const { return kind() == DebugTrap; } + bool isThunk() const { return kind() == FarJumpIsland; } + + // Functions, import exits, trap exits and JitEntry stubs have standard + // callable prologues and epilogues. Asynchronous frame iteration needs to + // know the offset of the return instruction to calculate the frame pointer. + + bool hasReturn() const { + return isFunction() || isImportExit() || isDebugTrap() || isJitEntry(); + } + uint32_t ret() const { + MOZ_ASSERT(hasReturn()); + return ret_; + } + + // Functions, export stubs and import stubs all have an associated function + // index. + + bool isJitEntry() const { return kind() == JitEntry; } + bool isInterpEntry() const { return kind() == InterpEntry; } + bool isEntry() const { return isInterpEntry() || isJitEntry(); } + bool hasFuncIndex() const { + return isFunction() || isImportExit() || isEntry(); + } + uint32_t funcIndex() const { + MOZ_ASSERT(hasFuncIndex()); + return u.funcIndex_; + } + + // TrapExit CodeRanges have a Trap field. + + Trap trap() const { + MOZ_ASSERT(isTrapExit()); + return u.trap_; + } + + // Function CodeRanges have two entry points: one for normal calls (with a + // known signature) and one for table calls (which involves dynamic + // signature checking). + + uint32_t funcCheckedCallEntry() const { + MOZ_ASSERT(isFunction()); + // not all functions have the checked call prologue; + // see GenerateFunctionPrologue + MOZ_ASSERT(u.func.beginToUncheckedCallEntry_ != 0); + return begin_; + } + uint32_t funcUncheckedCallEntry() const { + MOZ_ASSERT(isFunction()); + return begin_ + u.func.beginToUncheckedCallEntry_; + } + uint32_t funcTierEntry() const { + MOZ_ASSERT(isFunction()); + return begin_ + u.func.beginToTierEntry_; + } + uint32_t funcLineOrBytecode() const { + MOZ_ASSERT(isFunction()); + return u.func.lineOrBytecode_; + } + + // A sorted array of CodeRanges can be looked up via BinarySearch and + // OffsetInCode. + + struct OffsetInCode { + size_t offset; + explicit OffsetInCode(size_t offset) : offset(offset) {} + bool operator==(const CodeRange& rhs) const { + return offset >= rhs.begin() && offset < rhs.end(); + } + bool operator<(const CodeRange& rhs) const { return offset < rhs.begin(); } + }; +}; + +WASM_DECLARE_CACHEABLE_POD(CodeRange); +WASM_DECLARE_POD_VECTOR(CodeRange, CodeRangeVector) + +extern const CodeRange* LookupInSorted(const CodeRangeVector& codeRanges, + CodeRange::OffsetInCode target); + +// While the frame-pointer chain allows the stack to be unwound without +// metadata, Error.stack still needs to know the line/column of every call in +// the chain. A CallSiteDesc describes a single callsite to which CallSite adds +// the metadata necessary to walk up to the next frame. Lastly CallSiteAndTarget +// adds the function index of the callee. + +class CallSiteDesc { + static constexpr size_t LINE_OR_BYTECODE_BITS_SIZE = 28; + uint32_t lineOrBytecode_ : LINE_OR_BYTECODE_BITS_SIZE; + uint32_t kind_ : 4; + + WASM_CHECK_CACHEABLE_POD(lineOrBytecode_, kind_); + + public: + static constexpr uint32_t MAX_LINE_OR_BYTECODE_VALUE = + (1 << LINE_OR_BYTECODE_BITS_SIZE) - 1; + + enum Kind { + Func, // pc-relative call to a specific function + Import, // wasm import call + Indirect, // dynamic callee called via register, context on stack + IndirectFast, // dynamically determined to be same-instance + FuncRef, // call using direct function reference + FuncRefFast, // call using direct function reference within same-instance + Symbolic, // call to a single symbolic callee + EnterFrame, // call to a enter frame handler + LeaveFrame, // call to a leave frame handler + Breakpoint // call to instruction breakpoint + }; + CallSiteDesc() : lineOrBytecode_(0), kind_(0) {} + explicit CallSiteDesc(Kind kind) : lineOrBytecode_(0), kind_(kind) { + MOZ_ASSERT(kind == Kind(kind_)); + } + CallSiteDesc(uint32_t lineOrBytecode, Kind kind) + : lineOrBytecode_(lineOrBytecode), kind_(kind) { + MOZ_ASSERT(kind == Kind(kind_)); + MOZ_ASSERT(lineOrBytecode == lineOrBytecode_); + } + CallSiteDesc(BytecodeOffset bytecodeOffset, Kind kind) + : lineOrBytecode_(bytecodeOffset.offset()), kind_(kind) { + MOZ_ASSERT(kind == Kind(kind_)); + MOZ_ASSERT(bytecodeOffset.offset() == lineOrBytecode_); + } + uint32_t lineOrBytecode() const { return lineOrBytecode_; } + Kind kind() const { return Kind(kind_); } + bool isImportCall() const { return kind() == CallSiteDesc::Import; } + bool isIndirectCall() const { return kind() == CallSiteDesc::Indirect; } + bool isFuncRefCall() const { return kind() == CallSiteDesc::FuncRef; } + bool mightBeCrossInstance() const { + return isImportCall() || isIndirectCall() || isFuncRefCall(); + } +}; + +static_assert(js::wasm::MaxFunctionBytes <= + CallSiteDesc::MAX_LINE_OR_BYTECODE_VALUE); + +WASM_DECLARE_CACHEABLE_POD(CallSiteDesc); + +class CallSite : public CallSiteDesc { + uint32_t returnAddressOffset_; + + WASM_CHECK_CACHEABLE_POD_WITH_PARENT(CallSiteDesc, returnAddressOffset_); + + public: + CallSite() : returnAddressOffset_(0) {} + + CallSite(CallSiteDesc desc, uint32_t returnAddressOffset) + : CallSiteDesc(desc), returnAddressOffset_(returnAddressOffset) {} + + void offsetBy(uint32_t delta) { returnAddressOffset_ += delta; } + uint32_t returnAddressOffset() const { return returnAddressOffset_; } +}; + +WASM_DECLARE_CACHEABLE_POD(CallSite); +WASM_DECLARE_POD_VECTOR(CallSite, CallSiteVector) + +// A CallSiteTarget describes the callee of a CallSite, either a function or a +// trap exit. Although checked in debug builds, a CallSiteTarget doesn't +// officially know whether it targets a function or trap, relying on the Kind of +// the CallSite to discriminate. + +class CallSiteTarget { + uint32_t packed_; + + WASM_CHECK_CACHEABLE_POD(packed_); +#ifdef DEBUG + enum Kind { None, FuncIndex, TrapExit } kind_; + WASM_CHECK_CACHEABLE_POD(kind_); +#endif + + public: + explicit CallSiteTarget() + : packed_(UINT32_MAX) +#ifdef DEBUG + , + kind_(None) +#endif + { + } + + explicit CallSiteTarget(uint32_t funcIndex) + : packed_(funcIndex) +#ifdef DEBUG + , + kind_(FuncIndex) +#endif + { + } + + explicit CallSiteTarget(Trap trap) + : packed_(uint32_t(trap)) +#ifdef DEBUG + , + kind_(TrapExit) +#endif + { + } + + uint32_t funcIndex() const { + MOZ_ASSERT(kind_ == FuncIndex); + return packed_; + } + + Trap trap() const { + MOZ_ASSERT(kind_ == TrapExit); + MOZ_ASSERT(packed_ < uint32_t(Trap::Limit)); + return Trap(packed_); + } +}; + +WASM_DECLARE_CACHEABLE_POD(CallSiteTarget); + +using CallSiteTargetVector = Vector; + +// TryNotes are stored in a vector that acts as an exception table for +// wasm try-catch blocks. These represent the information needed to take +// exception handling actions after a throw is executed. +struct TryNote { + private: + // Sentinel value to detect a try note that has not been given a try body. + static const uint32_t BEGIN_NONE = UINT32_MAX; + + // Begin code offset of the try body. + uint32_t begin_; + // Exclusive end code offset of the try body. + uint32_t end_; + // The code offset of the landing pad. + uint32_t entryPoint_; + // Track offset from frame of stack pointer. + uint32_t framePushed_; + + WASM_CHECK_CACHEABLE_POD(begin_, end_, entryPoint_, framePushed_); + + public: + explicit TryNote() + : begin_(BEGIN_NONE), end_(0), entryPoint_(0), framePushed_(0) {} + + // Returns whether a try note has been assigned a range for the try body. + bool hasTryBody() const { return begin_ != BEGIN_NONE; } + + // The code offset of the beginning of the try body. + uint32_t tryBodyBegin() const { return begin_; } + + // The code offset of the exclusive end of the try body. + uint32_t tryBodyEnd() const { return end_; } + + // Returns whether an offset is within this try note's body. + bool offsetWithinTryBody(uint32_t offset) const { + return offset > begin_ && offset <= end_; + } + + // The code offset of the entry to the landing pad. + uint32_t landingPadEntryPoint() const { return entryPoint_; } + + // The stack frame pushed amount at the entry to the landing pad. + uint32_t landingPadFramePushed() const { return framePushed_; } + + // Set the beginning of the try body. + void setTryBodyBegin(uint32_t begin) { + // There must not be a begin to the try body yet + MOZ_ASSERT(begin_ == BEGIN_NONE); + begin_ = begin; + } + + // Set the end of the try body. + void setTryBodyEnd(uint32_t end) { + // There must be a begin to the try body + MOZ_ASSERT(begin_ != BEGIN_NONE); + end_ = end; + // We do not allow empty try bodies + MOZ_ASSERT(end_ > begin_); + } + + // Set the entry point and frame pushed of the landing pad. + void setLandingPad(uint32_t entryPoint, uint32_t framePushed) { + entryPoint_ = entryPoint; + framePushed_ = framePushed; + } + + // Adjust all code offsets in this try note by a delta. + void offsetBy(uint32_t offset) { + begin_ += offset; + end_ += offset; + entryPoint_ += offset; + } + + bool operator<(const TryNote& other) const { + // Special case comparison with self. This avoids triggering the assertion + // about non-intersection below. This case can arise in std::sort. + if (this == &other) { + return false; + } + // Try notes must be properly nested without touching at begin and end + MOZ_ASSERT(end_ <= other.begin_ || begin_ >= other.end_ || + (begin_ > other.begin_ && end_ < other.end_) || + (other.begin_ > begin_ && other.end_ < end_)); + // A total order is therefore given solely by comparing end points. This + // order will be such that the first try note to intersect a point is the + // innermost try note for that point. + return end_ < other.end_; + } +}; + +WASM_DECLARE_CACHEABLE_POD(TryNote); +WASM_DECLARE_POD_VECTOR(TryNote, TryNoteVector) + +// CallIndirectId describes how to compile a call_indirect and matching +// signature check in the function prologue for a given function type. + +enum class CallIndirectIdKind { AsmJS, Immediate, Global, None }; + +class CallIndirectId { + CallIndirectIdKind kind_; + size_t bits_; + + CallIndirectId(CallIndirectIdKind kind, size_t bits) + : kind_(kind), bits_(bits) {} + + public: + CallIndirectId() : kind_(CallIndirectIdKind::None), bits_(0) {} + + // Get a CallIndirectId for an asm.js function which will generate a no-op + // checked call prologue. + static CallIndirectId forAsmJSFunc(); + + // Get the CallIndirectId for a function in a specific module. + static CallIndirectId forFunc(const ModuleEnvironment& moduleEnv, + uint32_t funcIndex); + + // Get the CallIndirectId for a function type in a specific module. + static CallIndirectId forFuncType(const ModuleEnvironment& moduleEnv, + uint32_t funcTypeIndex); + + CallIndirectIdKind kind() const { return kind_; } + bool isGlobal() const { return kind_ == CallIndirectIdKind::Global; } + + uint32_t immediate() const { + MOZ_ASSERT(kind_ == CallIndirectIdKind::Immediate); + return bits_; + } + uint32_t instanceDataOffset() const { + MOZ_ASSERT(kind_ == CallIndirectIdKind::Global); + return bits_; + } +}; + +// CalleeDesc describes how to compile one of the variety of asm.js/wasm calls. +// This is hoisted into WasmCodegenTypes.h for sharing between Ion and Baseline. + +class CalleeDesc { + public: + enum Which { + // Calls a function defined in the same module by its index. + Func, + + // Calls the import identified by the offset of its FuncImportInstanceData + // in + // thread-local data. + Import, + + // Calls a WebAssembly table (heterogeneous, index must be bounds + // checked, callee instance depends on TableDesc). + WasmTable, + + // Calls an asm.js table (homogeneous, masked index, same-instance). + AsmJSTable, + + // Call a C++ function identified by SymbolicAddress. + Builtin, + + // Like Builtin, but automatically passes Instance* as first argument. + BuiltinInstanceMethod, + + // Calls a function reference. + FuncRef, + }; + + private: + // which_ shall be initialized in the static constructors + MOZ_INIT_OUTSIDE_CTOR Which which_; + union U { + U() : funcIndex_(0) {} + uint32_t funcIndex_; + struct { + uint32_t instanceDataOffset_; + } import; + struct { + uint32_t instanceDataOffset_; + uint32_t minLength_; + Maybe maxLength_; + CallIndirectId callIndirectId_; + } table; + SymbolicAddress builtin_; + } u; + + public: + CalleeDesc() = default; + static CalleeDesc function(uint32_t funcIndex); + static CalleeDesc import(uint32_t instanceDataOffset); + static CalleeDesc wasmTable(const ModuleEnvironment& moduleEnv, + const TableDesc& desc, uint32_t tableIndex, + CallIndirectId callIndirectId); + static CalleeDesc asmJSTable(const ModuleEnvironment& moduleEnv, + uint32_t tableIndex); + static CalleeDesc builtin(SymbolicAddress callee); + static CalleeDesc builtinInstanceMethod(SymbolicAddress callee); + static CalleeDesc wasmFuncRef(); + Which which() const { return which_; } + uint32_t funcIndex() const { + MOZ_ASSERT(which_ == Func); + return u.funcIndex_; + } + uint32_t importInstanceDataOffset() const { + MOZ_ASSERT(which_ == Import); + return u.import.instanceDataOffset_; + } + bool isTable() const { return which_ == WasmTable || which_ == AsmJSTable; } + uint32_t tableLengthInstanceDataOffset() const { + MOZ_ASSERT(isTable()); + return u.table.instanceDataOffset_ + offsetof(TableInstanceData, length); + } + uint32_t tableFunctionBaseInstanceDataOffset() const { + MOZ_ASSERT(isTable()); + return u.table.instanceDataOffset_ + offsetof(TableInstanceData, elements); + } + CallIndirectId wasmTableSigId() const { + MOZ_ASSERT(which_ == WasmTable); + return u.table.callIndirectId_; + } + uint32_t wasmTableMinLength() const { + MOZ_ASSERT(which_ == WasmTable); + return u.table.minLength_; + } + Maybe wasmTableMaxLength() const { + MOZ_ASSERT(which_ == WasmTable); + return u.table.maxLength_; + } + SymbolicAddress builtin() const { + MOZ_ASSERT(which_ == Builtin || which_ == BuiltinInstanceMethod); + return u.builtin_; + } + bool isFuncRef() const { return which_ == FuncRef; } +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_codegen_types_h diff --git a/js/src/wasm/WasmCompile.cpp b/js/src/wasm/WasmCompile.cpp new file mode 100644 index 0000000000..3471de1ad2 --- /dev/null +++ b/js/src/wasm/WasmCompile.cpp @@ -0,0 +1,919 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmCompile.h" + +#include "mozilla/Maybe.h" + +#include + +#ifndef __wasi__ +# include "jit/ProcessExecutableMemory.h" +#endif + +#include "jit/FlushICache.h" +#include "util/Text.h" +#include "vm/HelperThreads.h" +#include "vm/Realm.h" +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmGenerator.h" +#include "wasm/WasmIonCompile.h" +#include "wasm/WasmOpIter.h" +#include "wasm/WasmProcess.h" +#include "wasm/WasmSignalHandlers.h" +#include "wasm/WasmValidate.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +uint32_t wasm::ObservedCPUFeatures() { + enum Arch { + X86 = 0x1, + X64 = 0x2, + ARM = 0x3, + MIPS = 0x4, + MIPS64 = 0x5, + ARM64 = 0x6, + LOONG64 = 0x7, + RISCV64 = 0x8, + ARCH_BITS = 3 + }; + +#if defined(JS_CODEGEN_X86) + MOZ_ASSERT(uint32_t(jit::CPUInfo::GetFingerprint()) <= + (UINT32_MAX >> ARCH_BITS)); + return X86 | (uint32_t(jit::CPUInfo::GetFingerprint()) << ARCH_BITS); +#elif defined(JS_CODEGEN_X64) + MOZ_ASSERT(uint32_t(jit::CPUInfo::GetFingerprint()) <= + (UINT32_MAX >> ARCH_BITS)); + return X64 | (uint32_t(jit::CPUInfo::GetFingerprint()) << ARCH_BITS); +#elif defined(JS_CODEGEN_ARM) + MOZ_ASSERT(jit::GetARMFlags() <= (UINT32_MAX >> ARCH_BITS)); + return ARM | (jit::GetARMFlags() << ARCH_BITS); +#elif defined(JS_CODEGEN_ARM64) + MOZ_ASSERT(jit::GetARM64Flags() <= (UINT32_MAX >> ARCH_BITS)); + return ARM64 | (jit::GetARM64Flags() << ARCH_BITS); +#elif defined(JS_CODEGEN_MIPS64) + MOZ_ASSERT(jit::GetMIPSFlags() <= (UINT32_MAX >> ARCH_BITS)); + return MIPS64 | (jit::GetMIPSFlags() << ARCH_BITS); +#elif defined(JS_CODEGEN_LOONG64) + MOZ_ASSERT(jit::GetLOONG64Flags() <= (UINT32_MAX >> ARCH_BITS)); + return LOONG64 | (jit::GetLOONG64Flags() << ARCH_BITS); +#elif defined(JS_CODEGEN_RISCV64) + MOZ_ASSERT(jit::GetRISCV64Flags() <= (UINT32_MAX >> ARCH_BITS)); + return RISCV64 | (jit::GetRISCV64Flags() << ARCH_BITS); +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) + return 0; +#else +# error "unknown architecture" +#endif +} + +FeatureArgs FeatureArgs::build(JSContext* cx, const FeatureOptions& options) { + FeatureArgs features; + +#define WASM_FEATURE(NAME, LOWER_NAME, ...) \ + features.LOWER_NAME = wasm::NAME##Available(cx); + JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE); +#undef WASM_FEATURE + + features.sharedMemory = + wasm::ThreadsAvailable(cx) ? Shareable::True : Shareable::False; + + features.simd = jit::JitSupportsWasmSimd(); + features.intrinsics = options.intrinsics; + + return features; +} + +SharedCompileArgs CompileArgs::build(JSContext* cx, + ScriptedCaller&& scriptedCaller, + const FeatureOptions& options, + CompileArgsError* error) { + bool baseline = BaselineAvailable(cx); + bool ion = IonAvailable(cx); + + // Debug information such as source view or debug traps will require + // additional memory and permanently stay in baseline code, so we try to + // only enable it when a developer actually cares: when the debugger tab + // is open. + bool debug = cx->realm() && cx->realm()->debuggerObservesWasm(); + + bool forceTiering = + cx->options().testWasmAwaitTier2() || JitOptions.wasmDelayTier2; + + // The Available() predicates should ensure no failure here, but + // when we're fuzzing we allow inconsistent switches and the check may thus + // fail. Let it go to a run-time error instead of crashing. + if (debug && ion) { + *error = CompileArgsError::NoCompiler; + return nullptr; + } + + if (forceTiering && !(baseline && ion)) { + // This can happen only in testing, and in this case we don't have a + // proper way to signal the error, so just silently override the default, + // instead of adding a skip-if directive to every test using debug/gc. + forceTiering = false; + } + + if (!(baseline || ion)) { + *error = CompileArgsError::NoCompiler; + return nullptr; + } + + CompileArgs* target = cx->new_(std::move(scriptedCaller)); + if (!target) { + *error = CompileArgsError::OutOfMemory; + return nullptr; + } + + target->baselineEnabled = baseline; + target->ionEnabled = ion; + target->debugEnabled = debug; + target->forceTiering = forceTiering; + target->features = FeatureArgs::build(cx, options); + + return target; +} + +SharedCompileArgs CompileArgs::buildForAsmJS(ScriptedCaller&& scriptedCaller) { + CompileArgs* target = js_new(std::move(scriptedCaller)); + if (!target) { + return nullptr; + } + + // AsmJS is deprecated and doesn't have mechanisms for experimental features, + // so we don't need to initialize the FeatureArgs. It also only targets the + // Ion backend and does not need WASM debug support since it is de-optimized + // to JS in that case. + target->ionEnabled = true; + target->debugEnabled = false; + + return target; +} + +SharedCompileArgs CompileArgs::buildAndReport(JSContext* cx, + ScriptedCaller&& scriptedCaller, + const FeatureOptions& options, + bool reportOOM) { + CompileArgsError error; + SharedCompileArgs args = + CompileArgs::build(cx, std::move(scriptedCaller), options, &error); + if (args) { + Log(cx, "available wasm compilers: tier1=%s tier2=%s", + args->baselineEnabled ? "baseline" : "none", + args->ionEnabled ? "ion" : "none"); + return args; + } + + switch (error) { + case CompileArgsError::NoCompiler: { + JS_ReportErrorASCII(cx, "no WebAssembly compiler available"); + break; + } + case CompileArgsError::OutOfMemory: { + // Most callers are required to return 'false' without reporting an OOM, + // so we make reporting it optional here. + if (reportOOM) { + ReportOutOfMemory(cx); + } + break; + } + } + return nullptr; +} + +/* + * [SMDOC] Tiered wasm compilation. + * + * "Tiered compilation" refers to the mechanism where we first compile the code + * with a fast non-optimizing compiler so that we can start running the code + * quickly, while in the background recompiling the code with the slower + * optimizing compiler. Code created by baseline is called "tier-1"; code + * created by the optimizing compiler is called "tier-2". When the tier-2 code + * is ready, we "tier up" the code by creating paths from tier-1 code into their + * tier-2 counterparts; this patching is performed as the program is running. + * + * ## Selecting the compilation mode + * + * When wasm bytecode arrives, we choose the compilation strategy based on + * switches and on aspects of the code and the hardware. If switches allow + * tiered compilation to happen (the normal case), the following logic applies. + * + * If the code is sufficiently large that tiered compilation would be beneficial + * but not so large that it might blow our compiled code budget and make + * compilation fail, we choose tiered compilation. Otherwise we go straight to + * optimized code. + * + * The expected benefit of tiering is computed by TieringBeneficial(), below, + * based on various estimated parameters of the hardware: ratios of object code + * to byte code, speed of the system, number of cores. + * + * ## Mechanics of tiering up; patching + * + * Every time control enters a tier-1 function, the function prologue loads its + * tiering pointer from the tiering jump table (see JumpTable in WasmCode.h) and + * jumps to it. + * + * Initially, an entry in the tiering table points to the instruction inside the + * tier-1 function that follows the jump instruction (hence the jump is an + * expensive nop). When the tier-2 compiler is finished, the table is patched + * racily to point into the tier-2 function at the correct prologue location + * (see loop near the end of Module::finishTier2()). As tier-2 compilation is + * performed at most once per Module, there is at most one such racy overwrite + * per table element during the lifetime of the Module. + * + * The effect of the patching is to cause the tier-1 function to jump to its + * tier-2 counterpart whenever the tier-1 function is called subsequently. That + * is, tier-1 code performs standard frame setup on behalf of whatever code it + * jumps to, and the target code (tier-1 or tier-2) allocates its own frame in + * whatever way it wants. + * + * The racy writing means that it is often nondeterministic whether tier-1 or + * tier-2 code is reached by any call during the tiering-up process; if F calls + * A and B in that order, it may reach tier-2 code for A and tier-1 code for B. + * If F is running concurrently on threads T1 and T2, T1 and T2 may see code + * from different tiers for either function. + * + * Note, tiering up also requires upgrading the jit-entry stubs so that they + * reference tier-2 code. The mechanics of this upgrading are described at + * WasmInstanceObject::getExportedFunction(). + * + * ## Current limitations of tiering + * + * Tiering is not always seamless. Partly, it is possible for a program to get + * stuck in tier-1 code. Partly, a function that has tiered up continues to + * force execution to go via tier-1 code to reach tier-2 code, paying for an + * additional jump and a slightly less optimized prologue than tier-2 code could + * have had on its own. + * + * Known tiering limitiations: + * + * - We can tier up only at function boundaries. If a tier-1 function has a + * long-running loop it will not tier up until it returns to its caller. If + * this loop never exits (a runloop in a worker, for example) then the + * function will never tier up. + * + * To do better, we need OSR. + * + * - Wasm Table entries are never patched during tier-up. A Table of funcref + * holds not a JSFunction pointer, but a (code*,instance*) pair of pointers. + * When a table.set operation is performed, the JSFunction value is decomposed + * and its code and instance pointers are stored in the table; subsequently, + * when a table.get operation is performed, the JSFunction value is + * reconstituted from its code pointer using fairly elaborate machinery. (The + * mechanics are the same also for the reflected JS operations on a + * WebAssembly.Table. For everything, see WasmTable.{cpp,h}.) The code pointer + * in the Table will always be the code pointer belonging to the best tier that + * was active at the time when that function was stored in that Table slot; in + * many cases, it will be tier-1 code. As a consequence, a call through a table + * will first enter tier-1 code and then jump to tier-2 code. + * + * To do better, we must update all the tables in the system when an instance + * tiers up. This is expected to be very hard. + * + * - Imported Wasm functions are never patched during tier-up. Imports are held + * in FuncImportInstanceData values in the instance, and for a wasm + * callee, what's stored is the raw code pointer into the best tier of the + * callee that was active at the time the import was resolved. That could be + * baseline code, and if it is, the situation is as for Table entries: a call + * to an import will always go via that import's tier-1 code, which will tier + * up with an indirect jump. + * + * To do better, we must update all the import tables in the system that + * import functions from instances whose modules have tiered up. This is + * expected to be hard. + */ + +// Classify the current system as one of a set of recognizable classes. This +// really needs to get our tier-1 systems right. +// +// TODO: We don't yet have a good measure of how fast a system is. We +// distinguish between mobile and desktop because these are very different kinds +// of systems, but we could further distinguish between low / medium / high end +// within those major classes. If we do so, then constants below would be +// provided for each (class, architecture, system-tier) combination, not just +// (class, architecture) as now. +// +// CPU clock speed is not by itself a good predictor of system performance, as +// there are high-performance systems with slow clocks (recent Intel) and +// low-performance systems with fast clocks (older AMD). We can also use +// physical memory, core configuration, OS details, CPU class and family, and +// CPU manufacturer to disambiguate. + +enum class SystemClass { + DesktopX86, + DesktopX64, + DesktopUnknown32, + DesktopUnknown64, + MobileX86, + MobileArm32, + MobileArm64, + MobileUnknown32, + MobileUnknown64 +}; + +static SystemClass ClassifySystem() { + bool isDesktop; + +#if defined(ANDROID) || defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + isDesktop = false; +#else + isDesktop = true; +#endif + + if (isDesktop) { +#if defined(JS_CODEGEN_X64) + return SystemClass::DesktopX64; +#elif defined(JS_CODEGEN_X86) + return SystemClass::DesktopX86; +#elif defined(JS_64BIT) + return SystemClass::DesktopUnknown64; +#else + return SystemClass::DesktopUnknown32; +#endif + } else { +#if defined(JS_CODEGEN_X86) + return SystemClass::MobileX86; +#elif defined(JS_CODEGEN_ARM) + return SystemClass::MobileArm32; +#elif defined(JS_CODEGEN_ARM64) + return SystemClass::MobileArm64; +#elif defined(JS_64BIT) + return SystemClass::MobileUnknown64; +#else + return SystemClass::MobileUnknown32; +#endif + } +} + +// Code sizes in machine code bytes per bytecode byte, again empirical except +// where marked. +// +// The Ion estimate for ARM64 is the measured Baseline value scaled by a +// plausible factor for optimized code. + +static const double x64Tox86Inflation = 1.25; + +static const double x64IonBytesPerBytecode = 2.45; +static const double x86IonBytesPerBytecode = + x64IonBytesPerBytecode * x64Tox86Inflation; +static const double arm32IonBytesPerBytecode = 3.3; +static const double arm64IonBytesPerBytecode = 3.0 / 1.4; // Estimate + +static const double x64BaselineBytesPerBytecode = x64IonBytesPerBytecode * 1.43; +static const double x86BaselineBytesPerBytecode = + x64BaselineBytesPerBytecode * x64Tox86Inflation; +static const double arm32BaselineBytesPerBytecode = + arm32IonBytesPerBytecode * 1.39; +static const double arm64BaselineBytesPerBytecode = 3.0; + +static double OptimizedBytesPerBytecode(SystemClass cls) { + switch (cls) { + case SystemClass::DesktopX86: + case SystemClass::MobileX86: + case SystemClass::DesktopUnknown32: + return x86IonBytesPerBytecode; + case SystemClass::DesktopX64: + case SystemClass::DesktopUnknown64: + return x64IonBytesPerBytecode; + case SystemClass::MobileArm32: + case SystemClass::MobileUnknown32: + return arm32IonBytesPerBytecode; + case SystemClass::MobileArm64: + case SystemClass::MobileUnknown64: + return arm64IonBytesPerBytecode; + default: + MOZ_CRASH(); + } +} + +static double BaselineBytesPerBytecode(SystemClass cls) { + switch (cls) { + case SystemClass::DesktopX86: + case SystemClass::MobileX86: + case SystemClass::DesktopUnknown32: + return x86BaselineBytesPerBytecode; + case SystemClass::DesktopX64: + case SystemClass::DesktopUnknown64: + return x64BaselineBytesPerBytecode; + case SystemClass::MobileArm32: + case SystemClass::MobileUnknown32: + return arm32BaselineBytesPerBytecode; + case SystemClass::MobileArm64: + case SystemClass::MobileUnknown64: + return arm64BaselineBytesPerBytecode; + default: + MOZ_CRASH(); + } +} + +double wasm::EstimateCompiledCodeSize(Tier tier, size_t bytecodeSize) { + SystemClass cls = ClassifySystem(); + switch (tier) { + case Tier::Baseline: + return double(bytecodeSize) * BaselineBytesPerBytecode(cls); + case Tier::Optimized: + return double(bytecodeSize) * OptimizedBytesPerBytecode(cls); + } + MOZ_CRASH("bad tier"); +} + +// If parallel Ion compilation is going to take longer than this, we should +// tier. + +static const double tierCutoffMs = 10; + +// Compilation rate values are empirical except when noted, the reference +// systems are: +// +// Late-2013 MacBook Pro (2.6GHz 4 x hyperthreaded Haswell, Mac OS X) +// Late-2015 Nexus 5X (1.4GHz 4 x Cortex-A53 + 1.8GHz 2 x Cortex-A57, Android) +// Ca-2016 SoftIron Overdrive 1000 (1.7GHz 4 x Cortex-A57, Fedora) +// +// The rates are always per core. +// +// The estimate for ARM64 is the Baseline compilation rate on the SoftIron +// (because we have no Ion yet), divided by 5 to estimate Ion compile rate and +// then divided by 2 to make it more reasonable for consumer ARM64 systems. + +static const double x64IonBytecodesPerMs = 2100; +static const double x86IonBytecodesPerMs = 1500; +static const double arm32IonBytecodesPerMs = 450; +static const double arm64IonBytecodesPerMs = 750; // Estimate + +// Tiering cutoff values: if code section sizes are below these values (when +// divided by the effective number of cores) we do not tier, because we guess +// that parallel Ion compilation will be fast enough. + +static const double x64DesktopTierCutoff = x64IonBytecodesPerMs * tierCutoffMs; +static const double x86DesktopTierCutoff = x86IonBytecodesPerMs * tierCutoffMs; +static const double x86MobileTierCutoff = x86DesktopTierCutoff / 2; // Guess +static const double arm32MobileTierCutoff = + arm32IonBytecodesPerMs * tierCutoffMs; +static const double arm64MobileTierCutoff = + arm64IonBytecodesPerMs * tierCutoffMs; + +static double CodesizeCutoff(SystemClass cls) { + switch (cls) { + case SystemClass::DesktopX86: + case SystemClass::DesktopUnknown32: + return x86DesktopTierCutoff; + case SystemClass::DesktopX64: + case SystemClass::DesktopUnknown64: + return x64DesktopTierCutoff; + case SystemClass::MobileX86: + return x86MobileTierCutoff; + case SystemClass::MobileArm32: + case SystemClass::MobileUnknown32: + return arm32MobileTierCutoff; + case SystemClass::MobileArm64: + case SystemClass::MobileUnknown64: + return arm64MobileTierCutoff; + default: + MOZ_CRASH(); + } +} + +// As the number of cores grows the effectiveness of each core dwindles (on the +// systems we care about for SpiderMonkey). +// +// The data are empirical, computed from the observed compilation time of the +// Tanks demo code on a variable number of cores. +// +// The heuristic may fail on NUMA systems where the core count is high but the +// performance increase is nil or negative once the program moves beyond one +// socket. However, few browser users have such systems. + +static double EffectiveCores(uint32_t cores) { + if (cores <= 3) { + return pow(cores, 0.9); + } + return pow(cores, 0.75); +} + +#ifndef JS_64BIT +// Don't tier if tiering will fill code memory to more to more than this +// fraction. + +static const double spaceCutoffPct = 0.9; +#endif + +// Figure out whether we should use tiered compilation or not. +static bool TieringBeneficial(uint32_t codeSize) { + uint32_t cpuCount = GetHelperThreadCPUCount(); + MOZ_ASSERT(cpuCount > 0); + + // It's mostly sensible not to background compile when there's only one + // hardware thread as we want foreground computation to have access to that. + // However, if wasm background compilation helper threads can be given lower + // priority then background compilation on single-core systems still makes + // some kind of sense. That said, this is a non-issue: as of September 2017 + // 1-core was down to 3.5% of our population and falling. + + if (cpuCount == 1) { + return false; + } + + // Compute the max number of threads available to do actual background + // compilation work. + + uint32_t workers = GetMaxWasmCompilationThreads(); + + // The number of cores we will use is bounded both by the CPU count and the + // worker count, since the worker count already takes this into account. + + uint32_t cores = workers; + + SystemClass cls = ClassifySystem(); + + // Ion compilation on available cores must take long enough to be worth the + // bother. + + double cutoffSize = CodesizeCutoff(cls); + double effectiveCores = EffectiveCores(cores); + + if ((codeSize / effectiveCores) < cutoffSize) { + return false; + } + + // Do not implement a size cutoff for 64-bit systems since the code size + // budget for 64 bit is so large that it will hardly ever be an issue. + // (Also the cutoff percentage might be different on 64-bit.) + +#ifndef JS_64BIT + // If the amount of executable code for baseline compilation jeopardizes the + // availability of executable memory for ion code then do not tier, for now. + // + // TODO: For now we consider this module in isolation. We should really + // worry about what else is going on in this process and might be filling up + // the code memory. It's like we need some kind of code memory reservation + // system or JIT compilation for large modules. + + double ionRatio = OptimizedBytesPerBytecode(cls); + double baselineRatio = BaselineBytesPerBytecode(cls); + double needMemory = codeSize * (ionRatio + baselineRatio); + double availMemory = LikelyAvailableExecutableMemory(); + double cutoff = spaceCutoffPct * MaxCodeBytesPerProcess; + + // If the sum of baseline and ion code makes us exceeds some set percentage + // of the executable memory then disable tiering. + + if ((MaxCodeBytesPerProcess - availMemory) + needMemory > cutoff) { + return false; + } +#endif + + return true; +} + +// Ensure that we have the non-compiler requirements to tier safely. +static bool PlatformCanTier() { + return CanUseExtraThreads() && jit::CanFlushExecutionContextForAllThreads(); +} + +CompilerEnvironment::CompilerEnvironment(const CompileArgs& args) + : state_(InitialWithArgs), args_(&args) {} + +CompilerEnvironment::CompilerEnvironment(CompileMode mode, Tier tier, + DebugEnabled debugEnabled) + : state_(InitialWithModeTierDebug), + mode_(mode), + tier_(tier), + debug_(debugEnabled) {} + +void CompilerEnvironment::computeParameters() { + MOZ_ASSERT(state_ == InitialWithModeTierDebug); + + state_ = Computed; +} + +void CompilerEnvironment::computeParameters(Decoder& d) { + MOZ_ASSERT(!isComputed()); + + if (state_ == InitialWithModeTierDebug) { + computeParameters(); + return; + } + + bool baselineEnabled = args_->baselineEnabled; + bool ionEnabled = args_->ionEnabled; + bool debugEnabled = args_->debugEnabled; + bool forceTiering = args_->forceTiering; + + bool hasSecondTier = ionEnabled; + MOZ_ASSERT_IF(debugEnabled, baselineEnabled); + MOZ_ASSERT_IF(forceTiering, baselineEnabled && hasSecondTier); + + // Various constraints in various places should prevent failure here. + MOZ_RELEASE_ASSERT(baselineEnabled || ionEnabled); + + uint32_t codeSectionSize = 0; + + SectionRange range; + if (StartsCodeSection(d.begin(), d.end(), &range)) { + codeSectionSize = range.size; + } + + if (baselineEnabled && hasSecondTier && + (TieringBeneficial(codeSectionSize) || forceTiering) && + PlatformCanTier()) { + mode_ = CompileMode::Tier1; + tier_ = Tier::Baseline; + } else { + mode_ = CompileMode::Once; + tier_ = hasSecondTier ? Tier::Optimized : Tier::Baseline; + } + + debug_ = debugEnabled ? DebugEnabled::True : DebugEnabled::False; + + state_ = Computed; +} + +template +static bool DecodeFunctionBody(DecoderT& d, ModuleGenerator& mg, + uint32_t funcIndex) { + uint32_t bodySize; + if (!d.readVarU32(&bodySize)) { + return d.fail("expected number of function body bytes"); + } + + if (bodySize > MaxFunctionBytes) { + return d.fail("function body too big"); + } + + const size_t offsetInModule = d.currentOffset(); + + // Skip over the function body; it will be validated by the compilation + // thread. + const uint8_t* bodyBegin; + if (!d.readBytes(bodySize, &bodyBegin)) { + return d.fail("function body length too big"); + } + + return mg.compileFuncDef(funcIndex, offsetInModule, bodyBegin, + bodyBegin + bodySize); +} + +template +static bool DecodeCodeSection(const ModuleEnvironment& env, DecoderT& d, + ModuleGenerator& mg) { + if (!env.codeSection) { + if (env.numFuncDefs() != 0) { + return d.fail("expected code section"); + } + + return mg.finishFuncDefs(); + } + + uint32_t numFuncDefs; + if (!d.readVarU32(&numFuncDefs)) { + return d.fail("expected function body count"); + } + + if (numFuncDefs != env.numFuncDefs()) { + return d.fail( + "function body count does not match function signature count"); + } + + for (uint32_t funcDefIndex = 0; funcDefIndex < numFuncDefs; funcDefIndex++) { + if (!DecodeFunctionBody(d, mg, env.numFuncImports + funcDefIndex)) { + return false; + } + } + + if (!d.finishSection(*env.codeSection, "code")) { + return false; + } + + return mg.finishFuncDefs(); +} + +SharedModule wasm::CompileBuffer(const CompileArgs& args, + const ShareableBytes& bytecode, + UniqueChars* error, + UniqueCharsVector* warnings, + JS::OptimizedEncodingListener* listener) { + Decoder d(bytecode.bytes, 0, error, warnings); + + ModuleEnvironment moduleEnv(args.features); + if (!moduleEnv.init() || !DecodeModuleEnvironment(d, &moduleEnv)) { + return nullptr; + } + CompilerEnvironment compilerEnv(args); + compilerEnv.computeParameters(d); + + ModuleGenerator mg(args, &moduleEnv, &compilerEnv, nullptr, error, warnings); + if (!mg.init(nullptr)) { + return nullptr; + } + + if (!DecodeCodeSection(moduleEnv, d, mg)) { + return nullptr; + } + + if (!DecodeModuleTail(d, &moduleEnv)) { + return nullptr; + } + + return mg.finishModule(bytecode, listener); +} + +bool wasm::CompileTier2(const CompileArgs& args, const Bytes& bytecode, + const Module& module, UniqueChars* error, + UniqueCharsVector* warnings, Atomic* cancelled) { + Decoder d(bytecode, 0, error); + + ModuleEnvironment moduleEnv(args.features); + if (!moduleEnv.init() || !DecodeModuleEnvironment(d, &moduleEnv)) { + return false; + } + CompilerEnvironment compilerEnv(CompileMode::Tier2, Tier::Optimized, + DebugEnabled::False); + compilerEnv.computeParameters(d); + + ModuleGenerator mg(args, &moduleEnv, &compilerEnv, cancelled, error, + warnings); + if (!mg.init(nullptr)) { + return false; + } + + if (!DecodeCodeSection(moduleEnv, d, mg)) { + return false; + } + + if (!DecodeModuleTail(d, &moduleEnv)) { + return false; + } + + return mg.finishTier2(module); +} + +class StreamingDecoder { + Decoder d_; + const ExclusiveBytesPtr& codeBytesEnd_; + const Atomic& cancelled_; + + public: + StreamingDecoder(const ModuleEnvironment& env, const Bytes& begin, + const ExclusiveBytesPtr& codeBytesEnd, + const Atomic& cancelled, UniqueChars* error, + UniqueCharsVector* warnings) + : d_(begin, env.codeSection->start, error, warnings), + codeBytesEnd_(codeBytesEnd), + cancelled_(cancelled) {} + + bool fail(const char* msg) { return d_.fail(msg); } + + bool done() const { return d_.done(); } + + size_t currentOffset() const { return d_.currentOffset(); } + + bool waitForBytes(size_t numBytes) { + numBytes = std::min(numBytes, d_.bytesRemain()); + const uint8_t* requiredEnd = d_.currentPosition() + numBytes; + auto codeBytesEnd = codeBytesEnd_.lock(); + while (codeBytesEnd < requiredEnd) { + if (cancelled_) { + return false; + } + codeBytesEnd.wait(); + } + return true; + } + + bool readVarU32(uint32_t* u32) { + return waitForBytes(MaxVarU32DecodedBytes) && d_.readVarU32(u32); + } + + bool readBytes(size_t size, const uint8_t** begin) { + return waitForBytes(size) && d_.readBytes(size, begin); + } + + bool finishSection(const SectionRange& range, const char* name) { + return d_.finishSection(range, name); + } +}; + +static SharedBytes CreateBytecode(const Bytes& env, const Bytes& code, + const Bytes& tail, UniqueChars* error) { + size_t size = env.length() + code.length() + tail.length(); + if (size > MaxModuleBytes) { + *error = DuplicateString("module too big"); + return nullptr; + } + + MutableBytes bytecode = js_new(); + if (!bytecode || !bytecode->bytes.resize(size)) { + return nullptr; + } + + uint8_t* p = bytecode->bytes.begin(); + + memcpy(p, env.begin(), env.length()); + p += env.length(); + + memcpy(p, code.begin(), code.length()); + p += code.length(); + + memcpy(p, tail.begin(), tail.length()); + p += tail.length(); + + MOZ_ASSERT(p == bytecode->end()); + + return bytecode; +} + +SharedModule wasm::CompileStreaming( + const CompileArgs& args, const Bytes& envBytes, const Bytes& codeBytes, + const ExclusiveBytesPtr& codeBytesEnd, + const ExclusiveStreamEndData& exclusiveStreamEnd, + const Atomic& cancelled, UniqueChars* error, + UniqueCharsVector* warnings) { + CompilerEnvironment compilerEnv(args); + ModuleEnvironment moduleEnv(args.features); + if (!moduleEnv.init()) { + return nullptr; + } + + { + Decoder d(envBytes, 0, error, warnings); + + if (!DecodeModuleEnvironment(d, &moduleEnv)) { + return nullptr; + } + compilerEnv.computeParameters(d); + + if (!moduleEnv.codeSection) { + d.fail("unknown section before code section"); + return nullptr; + } + + MOZ_RELEASE_ASSERT(moduleEnv.codeSection->size == codeBytes.length()); + MOZ_RELEASE_ASSERT(d.done()); + } + + ModuleGenerator mg(args, &moduleEnv, &compilerEnv, &cancelled, error, + warnings); + if (!mg.init(nullptr)) { + return nullptr; + } + + { + StreamingDecoder d(moduleEnv, codeBytes, codeBytesEnd, cancelled, error, + warnings); + + if (!DecodeCodeSection(moduleEnv, d, mg)) { + return nullptr; + } + + MOZ_RELEASE_ASSERT(d.done()); + } + + { + auto streamEnd = exclusiveStreamEnd.lock(); + while (!streamEnd->reached) { + if (cancelled) { + return nullptr; + } + streamEnd.wait(); + } + } + + const StreamEndData& streamEnd = exclusiveStreamEnd.lock(); + const Bytes& tailBytes = *streamEnd.tailBytes; + + { + Decoder d(tailBytes, moduleEnv.codeSection->end(), error, warnings); + + if (!DecodeModuleTail(d, &moduleEnv)) { + return nullptr; + } + + MOZ_RELEASE_ASSERT(d.done()); + } + + SharedBytes bytecode = CreateBytecode(envBytes, codeBytes, tailBytes, error); + if (!bytecode) { + return nullptr; + } + + return mg.finishModule(*bytecode, streamEnd.tier2Listener); +} diff --git a/js/src/wasm/WasmCompile.h b/js/src/wasm/WasmCompile.h new file mode 100644 index 0000000000..2b07881eea --- /dev/null +++ b/js/src/wasm/WasmCompile.h @@ -0,0 +1,99 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_compile_h +#define wasm_compile_h + +#include "vm/Runtime.h" +#include "wasm/WasmModule.h" + +namespace JS { +class OptimizedEncodingListener; +} + +namespace js { +namespace wasm { + +// Return a uint32_t which captures the observed properties of the CPU that +// affect compilation. If code compiled now is to be serialized and executed +// later, the ObservedCPUFeatures() must be ensured to be the same. + +uint32_t ObservedCPUFeatures(); + +// Return the estimated compiled (machine) code size for the given bytecode size +// compiled at the given tier. + +double EstimateCompiledCodeSize(Tier tier, size_t bytecodeSize); + +// Compile the given WebAssembly bytecode with the given arguments into a +// wasm::Module. On success, the Module is returned. On failure, the returned +// SharedModule pointer is null and either: +// - *error points to a string description of the error +// - *error is null and the caller should report out-of-memory. + +SharedModule CompileBuffer(const CompileArgs& args, + const ShareableBytes& bytecode, UniqueChars* error, + UniqueCharsVector* warnings, + JS::OptimizedEncodingListener* listener = nullptr); + +// Attempt to compile the second tier of the given wasm::Module. + +bool CompileTier2(const CompileArgs& args, const Bytes& bytecode, + const Module& module, UniqueChars* error, + UniqueCharsVector* warnings, Atomic* cancelled); + +// Compile the given WebAssembly module which has been broken into three +// partitions: +// - envBytes contains a complete ModuleEnvironment that has already been +// copied in from the stream. +// - codeBytes is pre-sized to hold the complete code section when the stream +// completes. +// - The range [codeBytes.begin(), codeBytesEnd) contains the bytes currently +// read from the stream and codeBytesEnd will advance until either +// the stream is cancelled or codeBytesEnd == codeBytes.end(). +// - streamEnd contains the final information received after the code section: +// the remaining module bytecodes and maybe a JS::OptimizedEncodingListener. +// When the stream is successfully closed, streamEnd.reached is set. +// The ExclusiveWaitableData are notified when CompileStreaming() can make +// progress (i.e., codeBytesEnd advances or streamEnd.reached is set). +// If cancelled is set to true, compilation aborts and returns null. After +// cancellation is set, both ExclusiveWaitableData will be notified and so every +// wait() loop must check cancelled. + +using ExclusiveBytesPtr = ExclusiveWaitableData; + +struct StreamEndData { + bool reached; + const Bytes* tailBytes; + Tier2Listener tier2Listener; + + StreamEndData() : reached(false) {} +}; +using ExclusiveStreamEndData = ExclusiveWaitableData; + +SharedModule CompileStreaming(const CompileArgs& args, const Bytes& envBytes, + const Bytes& codeBytes, + const ExclusiveBytesPtr& codeBytesEnd, + const ExclusiveStreamEndData& streamEnd, + const Atomic& cancelled, UniqueChars* error, + UniqueCharsVector* warnings); + +} // namespace wasm +} // namespace js + +#endif // namespace wasm_compile_h diff --git a/js/src/wasm/WasmCompileArgs.h b/js/src/wasm/WasmCompileArgs.h new file mode 100644 index 0000000000..1cf45cb020 --- /dev/null +++ b/js/src/wasm/WasmCompileArgs.h @@ -0,0 +1,238 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_compile_args_h +#define wasm_compile_args_h + +#include "mozilla/RefPtr.h" + +#include "js/Utility.h" +#include "js/WasmFeatures.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmShareable.h" + +namespace js { +namespace wasm { + +enum class Shareable { False, True }; + +// Code can be compiled either with the Baseline compiler or the Ion compiler, +// and tier-variant data are tagged with the Tier value. +// +// A tier value is used to request tier-variant aspects of code, metadata, or +// linkdata. The tiers are normally explicit (Baseline and Ion); implicit tiers +// can be obtained through accessors on Code objects (eg, stableTier). + +enum class Tier { + Baseline, + Debug = Baseline, + Optimized, + Serialized = Optimized +}; + +// Iterator over tiers present in a tiered data structure. + +class Tiers { + Tier t_[2]; + uint32_t n_; + + public: + explicit Tiers() { n_ = 0; } + explicit Tiers(Tier t) { + t_[0] = t; + n_ = 1; + } + explicit Tiers(Tier t, Tier u) { + MOZ_ASSERT(t != u); + t_[0] = t; + t_[1] = u; + n_ = 2; + } + + Tier* begin() { return t_; } + Tier* end() { return t_ + n_; } +}; + +// Describes per-compilation settings that are controlled by an options bag +// passed to compilation and validation functions. (Nonstandard extension +// available under prefs.) + +struct FeatureOptions { + FeatureOptions() : intrinsics(false) {} + + // Enables intrinsic opcodes, only set in WasmIntrinsic.cpp. + bool intrinsics; +}; + +// Describes the features that control wasm compilation. + +struct FeatureArgs { + FeatureArgs() + : +#define WASM_FEATURE(NAME, LOWER_NAME, ...) LOWER_NAME(false), + JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE) +#undef WASM_FEATURE + sharedMemory(Shareable::False), + simd(false), + intrinsics(false) { + } + FeatureArgs(const FeatureArgs&) = default; + FeatureArgs& operator=(const FeatureArgs&) = default; + FeatureArgs(FeatureArgs&&) = default; + + static FeatureArgs build(JSContext* cx, const FeatureOptions& options); + +#define WASM_FEATURE(NAME, LOWER_NAME, ...) bool LOWER_NAME; + JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE) +#undef WASM_FEATURE + + Shareable sharedMemory; + bool simd; + bool intrinsics; +}; + +// Describes the JS scripted caller of a request to compile a wasm module. + +struct ScriptedCaller { + UniqueChars filename; // UTF-8 encoded + bool filenameIsURL; + unsigned line; + + ScriptedCaller() : filenameIsURL(false), line(0) {} +}; + +// Describes the reasons we cannot compute compile args + +enum class CompileArgsError { + OutOfMemory, + NoCompiler, +}; + +// Describes all the parameters that control wasm compilation. + +struct CompileArgs; +using MutableCompileArgs = RefPtr; +using SharedCompileArgs = RefPtr; + +struct CompileArgs : ShareableBase { + ScriptedCaller scriptedCaller; + UniqueChars sourceMapURL; + + bool baselineEnabled; + bool ionEnabled; + bool debugEnabled; + bool forceTiering; + + FeatureArgs features; + + // CompileArgs has several constructors: + // + // - two through factory functions `build`/`buildAndReport`, which checks + // that flags are consistent with each other, and optionally reports any + // errors. + // - the 'buildForAsmJS' one, which uses the appropriate configuration for + // legacy asm.js code. + // - one that gives complete access to underlying fields. + // + // You should use the factory functions in general, unless you have a very + // good reason (i.e. no JSContext around and you know which flags have been + // used). + + static SharedCompileArgs build(JSContext* cx, ScriptedCaller&& scriptedCaller, + const FeatureOptions& options, + CompileArgsError* error); + static SharedCompileArgs buildForAsmJS(ScriptedCaller&& scriptedCaller); + static SharedCompileArgs buildAndReport(JSContext* cx, + ScriptedCaller&& scriptedCaller, + const FeatureOptions& options, + bool reportOOM = false); + + explicit CompileArgs(ScriptedCaller&& scriptedCaller) + : scriptedCaller(std::move(scriptedCaller)), + baselineEnabled(false), + ionEnabled(false), + debugEnabled(false), + forceTiering(false) {} +}; + +// CompilerEnvironment holds any values that will be needed to compute +// compilation parameters once the module's feature opt-in sections have been +// parsed. +// +// Subsequent to construction a computeParameters() call will compute the final +// compilation parameters, and the object can then be queried for their values. + +struct CompileArgs; +class Decoder; + +struct CompilerEnvironment { + // The object starts in one of two "initial" states; computeParameters moves + // it into the "computed" state. + enum State { InitialWithArgs, InitialWithModeTierDebug, Computed }; + + State state_; + union { + // Value if the state_ == InitialWithArgs. + const CompileArgs* args_; + + // Value in the other two states. + struct { + CompileMode mode_; + Tier tier_; + DebugEnabled debug_; + }; + }; + + public: + // Retain a reference to the CompileArgs. A subsequent computeParameters() + // will compute all parameters from the CompileArgs and additional values. + explicit CompilerEnvironment(const CompileArgs& args); + + // Save the provided values for mode, tier, and debug, and the initial value + // for gc/refTypes. A subsequent computeParameters() will compute the + // final value of gc/refTypes. + CompilerEnvironment(CompileMode mode, Tier tier, DebugEnabled debugEnabled); + + // Compute any remaining compilation parameters. + void computeParameters(Decoder& d); + + // Compute any remaining compilation parameters. Only use this method if + // the CompilerEnvironment was created with values for mode, tier, and + // debug. + void computeParameters(); + + bool isComputed() const { return state_ == Computed; } + CompileMode mode() const { + MOZ_ASSERT(isComputed()); + return mode_; + } + Tier tier() const { + MOZ_ASSERT(isComputed()); + return tier_; + } + DebugEnabled debug() const { + MOZ_ASSERT(isComputed()); + return debug_; + } + bool debugEnabled() const { return debug() == DebugEnabled::True; } +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_compile_args_h diff --git a/js/src/wasm/WasmConstants.h b/js/src/wasm/WasmConstants.h new file mode 100644 index 0000000000..11bbc46dd2 --- /dev/null +++ b/js/src/wasm/WasmConstants.h @@ -0,0 +1,1144 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_constants_h +#define wasm_constants_h + +#include + +#include "wasm/WasmIntrinsicGenerated.h" + +namespace js { +namespace wasm { + +static const uint32_t MagicNumber = 0x6d736100; // "\0asm" +static const uint32_t EncodingVersion = 0x01; + +enum class SectionId { + Custom = 0, + Type = 1, + Import = 2, + Function = 3, + Table = 4, + Memory = 5, + Global = 6, + Export = 7, + Start = 8, + Elem = 9, + Code = 10, + Data = 11, + DataCount = 12, + Tag = 13, +}; + +// WebAssembly type encodings are all single-byte negative SLEB128s, hence: +// forall tc:TypeCode. ((tc & SLEB128SignMask) == SLEB128SignBit +static const uint8_t SLEB128SignMask = 0xc0; +static const uint8_t SLEB128SignBit = 0x40; + +enum class TypeCode { + + // If more "simple primitive" (non-reference, non-constructor, + // non-special-purpose) types are added here then you MUST update + // LowestPrimitiveTypeCode, below. + + I32 = 0x7f, // SLEB128(-0x01) + I64 = 0x7e, // SLEB128(-0x02) + F32 = 0x7d, // SLEB128(-0x03) + F64 = 0x7c, // SLEB128(-0x04) + V128 = 0x7b, // SLEB128(-0x05) + + I8 = 0x7a, // SLEB128(-0x06) + I16 = 0x79, // SLEB128(-0x07) + + // A function pointer with any signature + FuncRef = 0x70, // SLEB128(-0x10) + + // A reference to any host value. + ExternRef = 0x6f, // SLEB128(-0x11) + + // A reference to any wasm gc value. + AnyRef = 0x6e, // SLEB128(-0x12) + + // A reference to a struct/array value. + EqRef = 0x6d, // SLEB128(-0x13) + + // Type constructor for nullable reference types. + NullableRef = 0x6c, // SLEB128(-0x14) + + // Type constructor for non-nullable reference types. + Ref = 0x6b, // SLEB128(-0x15) + + // A null reference in the extern hierarchy. + NullExternRef = 0x69, // SLEB128(-0x17) + + // A null reference in the func hierarchy. + NullFuncRef = 0x68, // SLEB128(-0x18) + + // A reference to any struct value. + StructRef = 0x67, // SLEB128(-0x19) + + // A reference to any array value. + ArrayRef = 0x66, // SLEB128(-0x1A) + + // A null reference in the any hierarchy. + NullAnyRef = 0x65, // SLEB128(-0x1B) + + // Type constructor for function types + Func = 0x60, // SLEB128(-0x20) + + // Type constructor for structure types - gc proposal + Struct = 0x5f, // SLEB128(-0x21) + + // Type constructor for array types - gc proposal + Array = 0x5e, // SLEB128(-0x22) + + // Value for non-nullable type present. + TableHasInitExpr = 0x40, + + // The 'empty' case of blocktype. + BlockVoid = 0x40, // SLEB128(-0x40) + + // Type constructor for recursion groups - gc proposal + RecGroup = 0x4f, + + // TODO: update wasm-tools to use the correct prefix + RecGroupOld = 0x45, + + // Type prefix for parent types - gc proposal + SubType = 0x50, + + Limit = 0x80 +}; + +// This is the lowest-valued TypeCode that is a primitive type, used in +// UnpackTypeCodeTypeAbstracted(). If primitive typecodes are added below any +// reference typecode then the logic in that function MUST change. + +static constexpr TypeCode LowestPrimitiveTypeCode = TypeCode::I16; + +// An arbitrary reference type used as the result of +// UnpackTypeCodeTypeAbstracted() when a value type is a reference. + +static constexpr TypeCode AbstractReferenceTypeCode = TypeCode::ExternRef; + +// A type code used to represent (ref null? T) whether or not the type +// is encoded with 'Ref' or 'NullableRef'. + +static constexpr TypeCode AbstractTypeRefCode = TypeCode::Ref; + +// A wasm::Trap represents a wasm-defined trap that can occur during execution +// which triggers a WebAssembly.RuntimeError. Generated code may jump to a Trap +// symbolically, passing the bytecode offset to report as the trap offset. The +// generated jump will be bound to a tiny stub which fills the offset and +// then jumps to a per-Trap shared stub at the end of the module. + +enum class Trap { + // The Unreachable opcode has been executed. + Unreachable, + // An integer arithmetic operation led to an overflow. + IntegerOverflow, + // Trying to coerce NaN to an integer. + InvalidConversionToInteger, + // Integer division by zero. + IntegerDivideByZero, + // Out of bounds on wasm memory accesses. + OutOfBounds, + // Unaligned on wasm atomic accesses; also used for non-standard ARM + // unaligned access faults. + UnalignedAccess, + // call_indirect to null. + IndirectCallToNull, + // call_indirect signature mismatch. + IndirectCallBadSig, + // Dereference null pointer in operation on (Ref T) + NullPointerDereference, + // Failed to cast a (Ref T) in a ref.cast instruction + BadCast, + + // The internal stack space was exhausted. For compatibility, this throws + // the same over-recursed error as JS. + StackOverflow, + + // The wasm execution has potentially run too long and the engine must call + // CheckForInterrupt(). This trap is resumable. + CheckInterrupt, + + // Signal an error that was reported in C++ code. + ThrowReported, + + Limit +}; + +// The representation of a null reference value throughout the compiler. + +static const intptr_t NULLREF_VALUE = intptr_t((void*)nullptr); + +enum class DefinitionKind { + Function = 0x00, + Table = 0x01, + Memory = 0x02, + Global = 0x03, + Tag = 0x04, +}; + +enum class GlobalTypeImmediate { IsMutable = 0x1, AllowedMask = 0x1 }; + +enum class LimitsFlags { + Default = 0x0, + HasMaximum = 0x1, + IsShared = 0x2, + IsI64 = 0x4, +}; + +enum class LimitsMask { + Table = uint8_t(LimitsFlags::HasMaximum), +#ifdef ENABLE_WASM_MEMORY64 + Memory = uint8_t(LimitsFlags::HasMaximum) | uint8_t(LimitsFlags::IsShared) | + uint8_t(LimitsFlags::IsI64), +#else + Memory = uint8_t(LimitsFlags::HasMaximum) | uint8_t(LimitsFlags::IsShared), +#endif +}; + +enum class DataSegmentKind { + Active = 0x00, + Passive = 0x01, + ActiveWithMemoryIndex = 0x02 +}; + +enum class ElemSegmentKind : uint32_t { + Active = 0x0, + Passive = 0x1, + ActiveWithTableIndex = 0x2, + Declared = 0x3, +}; + +enum class ElemSegmentPayload : uint32_t { + ExternIndex = 0x0, + ElemExpression = 0x4, +}; + +enum class TagKind { + Exception = 0x0, +}; + +enum class Op { + // Control flow operators + Unreachable = 0x00, + Nop = 0x01, + Block = 0x02, + Loop = 0x03, + If = 0x04, + Else = 0x05, + Try = 0x06, + Catch = 0x07, + Throw = 0x08, + Rethrow = 0x09, + End = 0x0b, + Br = 0x0c, + BrIf = 0x0d, + BrTable = 0x0e, + Return = 0x0f, + + // Call operators + Call = 0x10, + CallIndirect = 0x11, + CallRef = 0x14, + + // Additional exception operators + Delegate = 0x18, + CatchAll = 0x19, + + // Parametric operators + Drop = 0x1a, + SelectNumeric = 0x1b, + SelectTyped = 0x1c, + + // Variable access + LocalGet = 0x20, + LocalSet = 0x21, + LocalTee = 0x22, + GlobalGet = 0x23, + GlobalSet = 0x24, + TableGet = 0x25, // Reftypes, + TableSet = 0x26, // per proposal as of February 2019 + + // Memory-related operators + I32Load = 0x28, + I64Load = 0x29, + F32Load = 0x2a, + F64Load = 0x2b, + I32Load8S = 0x2c, + I32Load8U = 0x2d, + I32Load16S = 0x2e, + I32Load16U = 0x2f, + I64Load8S = 0x30, + I64Load8U = 0x31, + I64Load16S = 0x32, + I64Load16U = 0x33, + I64Load32S = 0x34, + I64Load32U = 0x35, + I32Store = 0x36, + I64Store = 0x37, + F32Store = 0x38, + F64Store = 0x39, + I32Store8 = 0x3a, + I32Store16 = 0x3b, + I64Store8 = 0x3c, + I64Store16 = 0x3d, + I64Store32 = 0x3e, + MemorySize = 0x3f, + MemoryGrow = 0x40, + + // Constants + I32Const = 0x41, + I64Const = 0x42, + F32Const = 0x43, + F64Const = 0x44, + + // Comparison operators + I32Eqz = 0x45, + I32Eq = 0x46, + I32Ne = 0x47, + I32LtS = 0x48, + I32LtU = 0x49, + I32GtS = 0x4a, + I32GtU = 0x4b, + I32LeS = 0x4c, + I32LeU = 0x4d, + I32GeS = 0x4e, + I32GeU = 0x4f, + I64Eqz = 0x50, + I64Eq = 0x51, + I64Ne = 0x52, + I64LtS = 0x53, + I64LtU = 0x54, + I64GtS = 0x55, + I64GtU = 0x56, + I64LeS = 0x57, + I64LeU = 0x58, + I64GeS = 0x59, + I64GeU = 0x5a, + F32Eq = 0x5b, + F32Ne = 0x5c, + F32Lt = 0x5d, + F32Gt = 0x5e, + F32Le = 0x5f, + F32Ge = 0x60, + F64Eq = 0x61, + F64Ne = 0x62, + F64Lt = 0x63, + F64Gt = 0x64, + F64Le = 0x65, + F64Ge = 0x66, + + // Numeric operators + I32Clz = 0x67, + I32Ctz = 0x68, + I32Popcnt = 0x69, + I32Add = 0x6a, + I32Sub = 0x6b, + I32Mul = 0x6c, + I32DivS = 0x6d, + I32DivU = 0x6e, + I32RemS = 0x6f, + I32RemU = 0x70, + I32And = 0x71, + I32Or = 0x72, + I32Xor = 0x73, + I32Shl = 0x74, + I32ShrS = 0x75, + I32ShrU = 0x76, + I32Rotl = 0x77, + I32Rotr = 0x78, + I64Clz = 0x79, + I64Ctz = 0x7a, + I64Popcnt = 0x7b, + I64Add = 0x7c, + I64Sub = 0x7d, + I64Mul = 0x7e, + I64DivS = 0x7f, + I64DivU = 0x80, + I64RemS = 0x81, + I64RemU = 0x82, + I64And = 0x83, + I64Or = 0x84, + I64Xor = 0x85, + I64Shl = 0x86, + I64ShrS = 0x87, + I64ShrU = 0x88, + I64Rotl = 0x89, + I64Rotr = 0x8a, + F32Abs = 0x8b, + F32Neg = 0x8c, + F32Ceil = 0x8d, + F32Floor = 0x8e, + F32Trunc = 0x8f, + F32Nearest = 0x90, + F32Sqrt = 0x91, + F32Add = 0x92, + F32Sub = 0x93, + F32Mul = 0x94, + F32Div = 0x95, + F32Min = 0x96, + F32Max = 0x97, + F32CopySign = 0x98, + F64Abs = 0x99, + F64Neg = 0x9a, + F64Ceil = 0x9b, + F64Floor = 0x9c, + F64Trunc = 0x9d, + F64Nearest = 0x9e, + F64Sqrt = 0x9f, + F64Add = 0xa0, + F64Sub = 0xa1, + F64Mul = 0xa2, + F64Div = 0xa3, + F64Min = 0xa4, + F64Max = 0xa5, + F64CopySign = 0xa6, + + // Conversions + I32WrapI64 = 0xa7, + I32TruncF32S = 0xa8, + I32TruncF32U = 0xa9, + I32TruncF64S = 0xaa, + I32TruncF64U = 0xab, + I64ExtendI32S = 0xac, + I64ExtendI32U = 0xad, + I64TruncF32S = 0xae, + I64TruncF32U = 0xaf, + I64TruncF64S = 0xb0, + I64TruncF64U = 0xb1, + F32ConvertI32S = 0xb2, + F32ConvertI32U = 0xb3, + F32ConvertI64S = 0xb4, + F32ConvertI64U = 0xb5, + F32DemoteF64 = 0xb6, + F64ConvertI32S = 0xb7, + F64ConvertI32U = 0xb8, + F64ConvertI64S = 0xb9, + F64ConvertI64U = 0xba, + F64PromoteF32 = 0xbb, + + // Reinterpretations + I32ReinterpretF32 = 0xbc, + I64ReinterpretF64 = 0xbd, + F32ReinterpretI32 = 0xbe, + F64ReinterpretI64 = 0xbf, + + // Sign extension + I32Extend8S = 0xc0, + I32Extend16S = 0xc1, + I64Extend8S = 0xc2, + I64Extend16S = 0xc3, + I64Extend32S = 0xc4, + + // Reference types + RefNull = 0xd0, + RefIsNull = 0xd1, + RefFunc = 0xd2, + + // Function references + RefAsNonNull = 0xd3, + BrOnNull = 0xd4, + + // GC (experimental) + RefEq = 0xd5, + + // Function references + BrOnNonNull = 0xd6, + + FirstPrefix = 0xfa, + GcPrefix = 0xfb, + MiscPrefix = 0xfc, + SimdPrefix = 0xfd, + ThreadPrefix = 0xfe, + MozPrefix = 0xff, + + Limit = 0x100 +}; + +inline bool IsPrefixByte(uint8_t b) { return b >= uint8_t(Op::FirstPrefix); } + +// Opcodes in the GC opcode space. +enum class GcOp { + // Structure operations + StructNew = 0x7, + StructNewDefault = 0x8, + StructGet = 0x03, + StructGetS = 0x04, + StructGetU = 0x05, + StructSet = 0x06, + + // Array operations + ArrayNew = 0x1b, + ArrayNewFixed = 0x1a, + ArrayNewDefault = 0x1c, + ArrayNewData = 0x1d, + // array.init_from_elem_static in V5 became array.new_elem in V6, changing + // opcodes in the process + ArrayInitFromElemStaticV5 = 0x10, + ArrayNewElem = 0x1f, + ArrayGet = 0x13, + ArrayGetS = 0x14, + ArrayGetU = 0x15, + ArraySet = 0x16, + ArrayLenWithTypeIndex = 0x17, + ArrayCopy = 0x18, + ArrayLen = 0x19, + + // Ref operations + RefTestV5 = 0x44, + RefCastV5 = 0x45, + BrOnCastV5 = 0x46, + BrOnCastHeapV5 = 0x42, + BrOnCastHeapNullV5 = 0x4a, + BrOnCastFailV5 = 0x47, + BrOnCastFailHeapV5 = 0x43, + BrOnCastFailHeapNullV5 = 0x4b, + RefTest = 0x40, + RefCast = 0x41, + RefTestNull = 0x48, + RefCastNull = 0x49, + BrOnCast = 0x4f, + + // Dart compatibility instruction + RefAsStructV5 = 0x59, + BrOnNonStructV5 = 0x64, + + // Extern/any coercion operations + ExternInternalize = 0x70, + ExternExternalize = 0x71, + + Limit +}; + +// Opcode list from the SIMD proposal post-renumbering in May, 2020. + +// Opcodes with suffix 'Experimental' are proposed but not standardized, and are +// compatible with those same opcodes in V8. No opcode labeled 'Experimental' +// will ship in a Release build where SIMD is enabled by default. + +enum class SimdOp { + V128Load = 0x00, + V128Load8x8S = 0x01, + V128Load8x8U = 0x02, + V128Load16x4S = 0x03, + V128Load16x4U = 0x04, + V128Load32x2S = 0x05, + V128Load32x2U = 0x06, + V128Load8Splat = 0x07, + V128Load16Splat = 0x08, + V128Load32Splat = 0x09, + V128Load64Splat = 0x0a, + V128Store = 0x0b, + V128Const = 0x0c, + I8x16Shuffle = 0x0d, + I8x16Swizzle = 0x0e, + I8x16Splat = 0x0f, + I16x8Splat = 0x10, + I32x4Splat = 0x11, + I64x2Splat = 0x12, + F32x4Splat = 0x13, + F64x2Splat = 0x14, + I8x16ExtractLaneS = 0x15, + I8x16ExtractLaneU = 0x16, + I8x16ReplaceLane = 0x17, + I16x8ExtractLaneS = 0x18, + I16x8ExtractLaneU = 0x19, + I16x8ReplaceLane = 0x1a, + I32x4ExtractLane = 0x1b, + I32x4ReplaceLane = 0x1c, + I64x2ExtractLane = 0x1d, + I64x2ReplaceLane = 0x1e, + F32x4ExtractLane = 0x1f, + F32x4ReplaceLane = 0x20, + F64x2ExtractLane = 0x21, + F64x2ReplaceLane = 0x22, + I8x16Eq = 0x23, + I8x16Ne = 0x24, + I8x16LtS = 0x25, + I8x16LtU = 0x26, + I8x16GtS = 0x27, + I8x16GtU = 0x28, + I8x16LeS = 0x29, + I8x16LeU = 0x2a, + I8x16GeS = 0x2b, + I8x16GeU = 0x2c, + I16x8Eq = 0x2d, + I16x8Ne = 0x2e, + I16x8LtS = 0x2f, + I16x8LtU = 0x30, + I16x8GtS = 0x31, + I16x8GtU = 0x32, + I16x8LeS = 0x33, + I16x8LeU = 0x34, + I16x8GeS = 0x35, + I16x8GeU = 0x36, + I32x4Eq = 0x37, + I32x4Ne = 0x38, + I32x4LtS = 0x39, + I32x4LtU = 0x3a, + I32x4GtS = 0x3b, + I32x4GtU = 0x3c, + I32x4LeS = 0x3d, + I32x4LeU = 0x3e, + I32x4GeS = 0x3f, + I32x4GeU = 0x40, + F32x4Eq = 0x41, + F32x4Ne = 0x42, + F32x4Lt = 0x43, + F32x4Gt = 0x44, + F32x4Le = 0x45, + F32x4Ge = 0x46, + F64x2Eq = 0x47, + F64x2Ne = 0x48, + F64x2Lt = 0x49, + F64x2Gt = 0x4a, + F64x2Le = 0x4b, + F64x2Ge = 0x4c, + V128Not = 0x4d, + V128And = 0x4e, + V128AndNot = 0x4f, + V128Or = 0x50, + V128Xor = 0x51, + V128Bitselect = 0x52, + V128AnyTrue = 0x53, + V128Load8Lane = 0x54, + V128Load16Lane = 0x55, + V128Load32Lane = 0x56, + V128Load64Lane = 0x57, + V128Store8Lane = 0x58, + V128Store16Lane = 0x59, + V128Store32Lane = 0x5a, + V128Store64Lane = 0x5b, + V128Load32Zero = 0x5c, + V128Load64Zero = 0x5d, + F32x4DemoteF64x2Zero = 0x5e, + F64x2PromoteLowF32x4 = 0x5f, + I8x16Abs = 0x60, + I8x16Neg = 0x61, + I8x16Popcnt = 0x62, + I8x16AllTrue = 0x63, + I8x16Bitmask = 0x64, + I8x16NarrowI16x8S = 0x65, + I8x16NarrowI16x8U = 0x66, + F32x4Ceil = 0x67, + F32x4Floor = 0x68, + F32x4Trunc = 0x69, + F32x4Nearest = 0x6a, + I8x16Shl = 0x6b, + I8x16ShrS = 0x6c, + I8x16ShrU = 0x6d, + I8x16Add = 0x6e, + I8x16AddSatS = 0x6f, + I8x16AddSatU = 0x70, + I8x16Sub = 0x71, + I8x16SubSatS = 0x72, + I8x16SubSatU = 0x73, + F64x2Ceil = 0x74, + F64x2Floor = 0x75, + I8x16MinS = 0x76, + I8x16MinU = 0x77, + I8x16MaxS = 0x78, + I8x16MaxU = 0x79, + F64x2Trunc = 0x7a, + I8x16AvgrU = 0x7b, + I16x8ExtaddPairwiseI8x16S = 0x7c, + I16x8ExtaddPairwiseI8x16U = 0x7d, + I32x4ExtaddPairwiseI16x8S = 0x7e, + I32x4ExtaddPairwiseI16x8U = 0x7f, + I16x8Abs = 0x80, + I16x8Neg = 0x81, + I16x8Q15MulrSatS = 0x82, + I16x8AllTrue = 0x83, + I16x8Bitmask = 0x84, + I16x8NarrowI32x4S = 0x85, + I16x8NarrowI32x4U = 0x86, + I16x8ExtendLowI8x16S = 0x87, + I16x8ExtendHighI8x16S = 0x88, + I16x8ExtendLowI8x16U = 0x89, + I16x8ExtendHighI8x16U = 0x8a, + I16x8Shl = 0x8b, + I16x8ShrS = 0x8c, + I16x8ShrU = 0x8d, + I16x8Add = 0x8e, + I16x8AddSatS = 0x8f, + I16x8AddSatU = 0x90, + I16x8Sub = 0x91, + I16x8SubSatS = 0x92, + I16x8SubSatU = 0x93, + F64x2Nearest = 0x94, + I16x8Mul = 0x95, + I16x8MinS = 0x96, + I16x8MinU = 0x97, + I16x8MaxS = 0x98, + I16x8MaxU = 0x99, + // Unused = 0x9a + I16x8AvgrU = 0x9b, + I16x8ExtmulLowI8x16S = 0x9c, + I16x8ExtmulHighI8x16S = 0x9d, + I16x8ExtmulLowI8x16U = 0x9e, + I16x8ExtmulHighI8x16U = 0x9f, + I32x4Abs = 0xa0, + I32x4Neg = 0xa1, + // Unused = 0xa2 + I32x4AllTrue = 0xa3, + I32x4Bitmask = 0xa4, + // Unused = 0xa5 + // Unused = 0xa6 + I32x4ExtendLowI16x8S = 0xa7, + I32x4ExtendHighI16x8S = 0xa8, + I32x4ExtendLowI16x8U = 0xa9, + I32x4ExtendHighI16x8U = 0xaa, + I32x4Shl = 0xab, + I32x4ShrS = 0xac, + I32x4ShrU = 0xad, + I32x4Add = 0xae, + // Unused = 0xaf + // Unused = 0xb0 + I32x4Sub = 0xb1, + // Unused = 0xb2 + // Unused = 0xb3 + // Unused = 0xb4 + I32x4Mul = 0xb5, + I32x4MinS = 0xb6, + I32x4MinU = 0xb7, + I32x4MaxS = 0xb8, + I32x4MaxU = 0xb9, + I32x4DotI16x8S = 0xba, + // Unused = 0xbb + I32x4ExtmulLowI16x8S = 0xbc, + I32x4ExtmulHighI16x8S = 0xbd, + I32x4ExtmulLowI16x8U = 0xbe, + I32x4ExtmulHighI16x8U = 0xbf, + I64x2Abs = 0xc0, + I64x2Neg = 0xc1, + // AnyTrue = 0xc2 + I64x2AllTrue = 0xc3, + I64x2Bitmask = 0xc4, + // Unused = 0xc5 + // Unused = 0xc6 + I64x2ExtendLowI32x4S = 0xc7, + I64x2ExtendHighI32x4S = 0xc8, + I64x2ExtendLowI32x4U = 0xc9, + I64x2ExtendHighI32x4U = 0xca, + I64x2Shl = 0xcb, + I64x2ShrS = 0xcc, + I64x2ShrU = 0xcd, + I64x2Add = 0xce, + // Unused = 0xcf + // Unused = 0xd0 + I64x2Sub = 0xd1, + // Unused = 0xd2 + // Unused = 0xd3 + // Unused = 0xd4 + I64x2Mul = 0xd5, + I64x2Eq = 0xd6, + I64x2Ne = 0xd7, + I64x2LtS = 0xd8, + I64x2GtS = 0xd9, + I64x2LeS = 0xda, + I64x2GeS = 0xdb, + I64x2ExtmulLowI32x4S = 0xdc, + I64x2ExtmulHighI32x4S = 0xdd, + I64x2ExtmulLowI32x4U = 0xde, + I64x2ExtmulHighI32x4U = 0xdf, + F32x4Abs = 0xe0, + F32x4Neg = 0xe1, + // Unused = 0xe2 + F32x4Sqrt = 0xe3, + F32x4Add = 0xe4, + F32x4Sub = 0xe5, + F32x4Mul = 0xe6, + F32x4Div = 0xe7, + F32x4Min = 0xe8, + F32x4Max = 0xe9, + F32x4PMin = 0xea, + F32x4PMax = 0xeb, + F64x2Abs = 0xec, + F64x2Neg = 0xed, + // Unused = 0xee + F64x2Sqrt = 0xef, + F64x2Add = 0xf0, + F64x2Sub = 0xf1, + F64x2Mul = 0xf2, + F64x2Div = 0xf3, + F64x2Min = 0xf4, + F64x2Max = 0xf5, + F64x2PMin = 0xf6, + F64x2PMax = 0xf7, + I32x4TruncSatF32x4S = 0xf8, + I32x4TruncSatF32x4U = 0xf9, + F32x4ConvertI32x4S = 0xfa, + F32x4ConvertI32x4U = 0xfb, + I32x4TruncSatF64x2SZero = 0xfc, + I32x4TruncSatF64x2UZero = 0xfd, + F64x2ConvertLowI32x4S = 0xfe, + F64x2ConvertLowI32x4U = 0xff, + I8x16RelaxedSwizzle = 0x100, + I32x4RelaxedTruncF32x4S = 0x101, + I32x4RelaxedTruncF32x4U = 0x102, + I32x4RelaxedTruncF64x2SZero = 0x103, + I32x4RelaxedTruncF64x2UZero = 0x104, + F32x4RelaxedFma = 0x105, + F32x4RelaxedFnma = 0x106, + F64x2RelaxedFma = 0x107, + F64x2RelaxedFnma = 0x108, + I8x16RelaxedLaneSelect = 0x109, + I16x8RelaxedLaneSelect = 0x10a, + I32x4RelaxedLaneSelect = 0x10b, + I64x2RelaxedLaneSelect = 0x10c, + F32x4RelaxedMin = 0x10d, + F32x4RelaxedMax = 0x10e, + F64x2RelaxedMin = 0x10f, + F64x2RelaxedMax = 0x110, + I16x8RelaxedQ15MulrS = 0x111, + I16x8DotI8x16I7x16S = 0x112, + I32x4DotI8x16I7x16AddS = 0x113, + + // Reserved for Relaxed SIMD = 0x114-0x12f + + // Unused = 0x130 and up + + // Mozilla extensions + MozPMADDUBSW = 0x201, + + Limit +}; + +// Opcodes in the "miscellaneous" opcode space. +enum class MiscOp { + // Saturating float-to-int conversions + I32TruncSatF32S = 0x00, + I32TruncSatF32U = 0x01, + I32TruncSatF64S = 0x02, + I32TruncSatF64U = 0x03, + I64TruncSatF32S = 0x04, + I64TruncSatF32U = 0x05, + I64TruncSatF64S = 0x06, + I64TruncSatF64U = 0x07, + + // Bulk memory operations, per proposal as of February 2019. + MemoryInit = 0x08, + DataDrop = 0x09, + MemoryCopy = 0x0a, + MemoryFill = 0x0b, + TableInit = 0x0c, + ElemDrop = 0x0d, + TableCopy = 0x0e, + + // Reftypes, per proposal as of February 2019. + TableGrow = 0x0f, + TableSize = 0x10, + TableFill = 0x11, + + MemoryDiscard = 0x12, + + Limit +}; + +// Opcodes from threads proposal as of June 30, 2017 +enum class ThreadOp { + // Wait and wake + Wake = 0x00, + I32Wait = 0x01, + I64Wait = 0x02, + Fence = 0x03, + + // Load and store + I32AtomicLoad = 0x10, + I64AtomicLoad = 0x11, + I32AtomicLoad8U = 0x12, + I32AtomicLoad16U = 0x13, + I64AtomicLoad8U = 0x14, + I64AtomicLoad16U = 0x15, + I64AtomicLoad32U = 0x16, + I32AtomicStore = 0x17, + I64AtomicStore = 0x18, + I32AtomicStore8U = 0x19, + I32AtomicStore16U = 0x1a, + I64AtomicStore8U = 0x1b, + I64AtomicStore16U = 0x1c, + I64AtomicStore32U = 0x1d, + + // Read-modify-write operations + I32AtomicAdd = 0x1e, + I64AtomicAdd = 0x1f, + I32AtomicAdd8U = 0x20, + I32AtomicAdd16U = 0x21, + I64AtomicAdd8U = 0x22, + I64AtomicAdd16U = 0x23, + I64AtomicAdd32U = 0x24, + + I32AtomicSub = 0x25, + I64AtomicSub = 0x26, + I32AtomicSub8U = 0x27, + I32AtomicSub16U = 0x28, + I64AtomicSub8U = 0x29, + I64AtomicSub16U = 0x2a, + I64AtomicSub32U = 0x2b, + + I32AtomicAnd = 0x2c, + I64AtomicAnd = 0x2d, + I32AtomicAnd8U = 0x2e, + I32AtomicAnd16U = 0x2f, + I64AtomicAnd8U = 0x30, + I64AtomicAnd16U = 0x31, + I64AtomicAnd32U = 0x32, + + I32AtomicOr = 0x33, + I64AtomicOr = 0x34, + I32AtomicOr8U = 0x35, + I32AtomicOr16U = 0x36, + I64AtomicOr8U = 0x37, + I64AtomicOr16U = 0x38, + I64AtomicOr32U = 0x39, + + I32AtomicXor = 0x3a, + I64AtomicXor = 0x3b, + I32AtomicXor8U = 0x3c, + I32AtomicXor16U = 0x3d, + I64AtomicXor8U = 0x3e, + I64AtomicXor16U = 0x3f, + I64AtomicXor32U = 0x40, + + I32AtomicXchg = 0x41, + I64AtomicXchg = 0x42, + I32AtomicXchg8U = 0x43, + I32AtomicXchg16U = 0x44, + I64AtomicXchg8U = 0x45, + I64AtomicXchg16U = 0x46, + I64AtomicXchg32U = 0x47, + + // CompareExchange + I32AtomicCmpXchg = 0x48, + I64AtomicCmpXchg = 0x49, + I32AtomicCmpXchg8U = 0x4a, + I32AtomicCmpXchg16U = 0x4b, + I64AtomicCmpXchg8U = 0x4c, + I64AtomicCmpXchg16U = 0x4d, + I64AtomicCmpXchg32U = 0x4e, + + Limit +}; + +enum class IntrinsicId { +// ------------------------------------------------------------------------ +// These are part/suffix of the MozOp::Intrinsic operators that are emitted +// internally when compiling intrinsic modules and are rejected by wasm +// validation. +// See wasm/WasmIntrinsic.yaml for the list. +#define DECL_INTRINSIC_OP(op, export, sa_name, abitype, entry, idx) \ + op = idx, // NOLINT + FOR_EACH_INTRINSIC(DECL_INTRINSIC_OP) +#undef DECL_INTRINSIC_OP + + // Op limit. + Limit +}; + +enum class MozOp { + // ------------------------------------------------------------------------ + // These operators are emitted internally when compiling asm.js and are + // rejected by wasm validation. They are prefixed by MozPrefix. + + // asm.js-specific operators. They start at 1 so as to check for + // uninitialized (zeroed) storage. + TeeGlobal = 0x01, + I32Min, + I32Max, + I32Neg, + I32BitNot, + I32Abs, + F32TeeStoreF64, + F64TeeStoreF32, + I32TeeStore8, + I32TeeStore16, + I64TeeStore8, + I64TeeStore16, + I64TeeStore32, + I32TeeStore, + I64TeeStore, + F32TeeStore, + F64TeeStore, + F64Mod, + F64SinNative, + F64SinFdlibm, + F64CosNative, + F64CosFdlibm, + F64TanNative, + F64TanFdlibm, + F64Asin, + F64Acos, + F64Atan, + F64Exp, + F64Log, + F64Pow, + F64Atan2, + + // asm.js-style call_indirect with the callee evaluated first. + OldCallDirect, + OldCallIndirect, + + // Intrinsic modules operations. The operator has argument leb u32 to specify + // particular operation id. See IntrinsicId above. + Intrinsic, + + Limit +}; + +struct OpBytes { + // b0 is a byte value but has a 16-bit representation to allow for a full + // 256-value range plus a sentinel Limit value. + uint16_t b0; + // b1 is a LEB128 value but 32 bits is enough for now. + uint32_t b1; + + explicit OpBytes(Op x) { + b0 = uint16_t(x); + b1 = 0; + } + OpBytes() = default; + + // Whether this opcode should have a breakpoint site inserted directly before + // the opcode in baseline when debugging. We use this as a heuristic to + // reduce the number of breakpoint sites. + bool shouldHaveBreakpoint() const { + switch (Op(b0)) { + // Block-like instructions don't get their own breakpoint site, a + // breakpoint can be used on instructions in the block. + case Op::Block: + case Op::Loop: + case Op::If: + case Op::Else: + case Op::Try: + case Op::Delegate: + case Op::Catch: + case Op::CatchAll: + case Op::End: + // Effect-less instructions without inputs are leaf nodes in expressions, + // a breakpoint can be used on instructions that consume these values. + case Op::LocalGet: + case Op::GlobalGet: + case Op::I32Const: + case Op::I64Const: + case Op::F32Const: + case Op::F64Const: + case Op::RefNull: + case Op::Drop: + return false; + default: + return true; + } + } +}; + +static const char NameSectionName[] = "name"; +static const char SourceMappingURLSectionName[] = "sourceMappingURL"; + +enum class NameType { Module = 0, Function = 1, Local = 2 }; + +enum class FieldFlags { Mutable = 0x01, AllowedMask = 0x01 }; + +enum class FieldWideningOp { None, Signed, Unsigned }; + +// The WebAssembly spec hard-codes the virtual page size to be 64KiB and +// requires the size of linear memory to always be a multiple of 64KiB. + +static const unsigned PageSize = 64 * 1024; +static const unsigned PageBits = 16; +static_assert(PageSize == (1u << PageBits)); + +static const unsigned PageMask = ((1u << PageBits) - 1); + +// These limits are agreed upon with other engines for consistency. + +static const unsigned MaxTypes = 1000000; +static const unsigned MaxFuncs = 1000000; +static const unsigned MaxTables = 100000; +static const unsigned MaxImports = 100000; +static const unsigned MaxExports = 100000; +static const unsigned MaxGlobals = 1000000; +static const unsigned MaxDataSegments = 100000; +static const unsigned MaxDataSegmentLengthPages = 16384; +static const unsigned MaxElemSegments = 10000000; +static const unsigned MaxElemSegmentLength = 10000000; +static const unsigned MaxTableLimitField = UINT32_MAX; +static const unsigned MaxTableLength = 10000000; +static const unsigned MaxLocals = 50000; +static const unsigned MaxParams = 1000; +static const unsigned MaxResults = 1000; +static const unsigned MaxStructFields = 2000; +static const uint64_t MaxMemory32LimitField = uint64_t(1) << 16; +static const uint64_t MaxMemory64LimitField = uint64_t(1) << 48; +static const unsigned MaxStringBytes = 100000; +static const unsigned MaxModuleBytes = 1024 * 1024 * 1024; +static const unsigned MaxFunctionBytes = 7654321; + +// These limits pertain to our WebAssembly implementation only, but may make +// sense to get into the shared limits spec eventually. + +static const unsigned MaxRecGroups = 1000000; +static const unsigned MaxSubTypingDepth = 31; +static const unsigned MaxTags = 1000000; + +// Maximum payload size, in bytes, of a gc-proposal Array. Puts it fairly +// close to 2^31 without exposing us to potential danger at the signed-i32 +// wraparound boundary. Note that gc-proposal Struct sizes are limited by +// MaxStructFields above. Some code assumes that the payload size will fit in +// a uint32_t, hence the static assert. +static const unsigned MaxArrayPayloadBytes = 1987654321; +static_assert(uint64_t(MaxArrayPayloadBytes) < + (uint64_t(1) << (8 * sizeof(uint32_t)))); + +// These limits pertain to our WebAssembly implementation only. + +static const unsigned MaxBrTableElems = 1000000; +static const unsigned MaxCodeSectionBytes = MaxModuleBytes; + +// 512KiB should be enough, considering how Rabaldr uses the stack and +// what the standard limits are: +// +// - 1,000 parameters +// - 50,000 locals +// - 10,000 values on the eval stack (not an official limit) +// +// At sizeof(int64) bytes per slot this works out to about 480KiB. + +static const unsigned MaxFrameSize = 512 * 1024; + +// Asserted by Decoder::readVarU32. + +static const unsigned MaxVarU32DecodedBytes = 5; + +// The CompileMode controls how compilation of a module is performed (notably, +// how many times we compile it). + +enum class CompileMode { Once, Tier1, Tier2 }; + +// Typed enum for whether debugging is enabled. + +enum class DebugEnabled { False, True }; + +} // namespace wasm +} // namespace js + +#endif // wasm_constants_h diff --git a/js/src/wasm/WasmContext.h b/js/src/wasm/WasmContext.h new file mode 100644 index 0000000000..5064401a81 --- /dev/null +++ b/js/src/wasm/WasmContext.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2020 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_context_h +#define wasm_context_h + +namespace js { +namespace wasm { + +// wasm::Context lives in JSContext and contains the wasm-related per-context +// state. + +class Context { + public: + Context() : triedToInstallSignalHandlers(false), haveSignalHandlers(false) {} + + // Used by wasm::EnsureThreadSignalHandlers(cx) to install thread signal + // handlers once per JSContext/thread. + bool triedToInstallSignalHandlers; + bool haveSignalHandlers; +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_context_h diff --git a/js/src/wasm/WasmDebug.cpp b/js/src/wasm/WasmDebug.cpp new file mode 100644 index 0000000000..66a59e826e --- /dev/null +++ b/js/src/wasm/WasmDebug.cpp @@ -0,0 +1,528 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmDebug.h" + +#include "mozilla/BinarySearch.h" + +#include "debugger/Debugger.h" +#include "ds/Sort.h" +#include "jit/MacroAssembler.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmValidate.h" + +#include "gc/GCContext-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::BinarySearchIf; + +DebugState::DebugState(const Code& code, const Module& module) + : code_(&code), + module_(&module), + enterFrameTrapsEnabled_(false), + enterAndLeaveFrameTrapsCounter_(0) { + MOZ_RELEASE_ASSERT(code.metadata().debugEnabled); + MOZ_RELEASE_ASSERT(code.hasTier(Tier::Debug)); +} + +void DebugState::trace(JSTracer* trc) { + for (auto iter = breakpointSites_.iter(); !iter.done(); iter.next()) { + WasmBreakpointSite* site = iter.get().value(); + site->trace(trc); + } +} + +void DebugState::finalize(JS::GCContext* gcx) { + for (auto iter = breakpointSites_.iter(); !iter.done(); iter.next()) { + WasmBreakpointSite* site = iter.get().value(); + site->delete_(gcx); + } +} + +static const uint32_t DefaultBinarySourceColumnNumber = 1; + +static const CallSite* SlowCallSiteSearchByOffset(const MetadataTier& metadata, + uint32_t offset) { + for (const CallSite& callSite : metadata.callSites) { + if (callSite.lineOrBytecode() == offset && + callSite.kind() == CallSiteDesc::Breakpoint) { + return &callSite; + } + } + return nullptr; +} + +bool DebugState::getLineOffsets(size_t lineno, Vector* offsets) { + const CallSite* callsite = + SlowCallSiteSearchByOffset(metadata(Tier::Debug), lineno); + return !(callsite && !offsets->append(lineno)); +} + +bool DebugState::getAllColumnOffsets(Vector* offsets) { + for (const CallSite& callSite : metadata(Tier::Debug).callSites) { + if (callSite.kind() != CallSite::Breakpoint) { + continue; + } + uint32_t offset = callSite.lineOrBytecode(); + if (!offsets->emplaceBack(offset, DefaultBinarySourceColumnNumber, + offset)) { + return false; + } + } + return true; +} + +bool DebugState::getOffsetLocation(uint32_t offset, size_t* lineno, + size_t* column) { + if (!SlowCallSiteSearchByOffset(metadata(Tier::Debug), offset)) { + return false; + } + *lineno = offset; + *column = DefaultBinarySourceColumnNumber; + return true; +} + +bool DebugState::stepModeEnabled(uint32_t funcIndex) const { + return stepperCounters_.lookup(funcIndex).found(); +} + +bool DebugState::incrementStepperCount(JSContext* cx, Instance* instance, + uint32_t funcIndex) { + StepperCounters::AddPtr p = stepperCounters_.lookupForAdd(funcIndex); + if (p) { + MOZ_ASSERT(p->value() > 0); + p->value()++; + return true; + } + + if (!stepperCounters_.add(p, funcIndex, 1)) { + ReportOutOfMemory(cx); + return false; + } + + enableDebuggingForFunction(instance, funcIndex); + enableDebugTrap(instance); + + return true; +} + +void DebugState::decrementStepperCount(JS::GCContext* gcx, Instance* instance, + uint32_t funcIndex) { + const CodeRange& codeRange = + codeRanges(Tier::Debug)[funcToCodeRangeIndex(funcIndex)]; + MOZ_ASSERT(codeRange.isFunction()); + + MOZ_ASSERT(!stepperCounters_.empty()); + StepperCounters::Ptr p = stepperCounters_.lookup(funcIndex); + MOZ_ASSERT(p); + if (--p->value()) { + return; + } + + stepperCounters_.remove(p); + + bool anyStepping = !stepperCounters_.empty(); + bool anyBreakpoints = !breakpointSites_.empty(); + bool anyEnterAndLeave = enterAndLeaveFrameTrapsCounter_ > 0; + + bool keepDebugging = false; + for (const CallSite& callSite : callSites(Tier::Debug)) { + if (callSite.kind() != CallSite::Breakpoint) { + continue; + } + uint32_t offset = callSite.returnAddressOffset(); + if (codeRange.begin() <= offset && offset <= codeRange.end()) { + keepDebugging = keepDebugging || breakpointSites_.has(offset); + } + } + + if (!keepDebugging && !anyEnterAndLeave) { + disableDebuggingForFunction(instance, funcIndex); + if (!anyStepping && !anyBreakpoints) { + disableDebugTrap(instance); + } + } +} + +bool DebugState::hasBreakpointTrapAtOffset(uint32_t offset) { + return SlowCallSiteSearchByOffset(metadata(Tier::Debug), offset); +} + +void DebugState::toggleBreakpointTrap(JSRuntime* rt, Instance* instance, + uint32_t offset, bool enabled) { + const CallSite* callSite = + SlowCallSiteSearchByOffset(metadata(Tier::Debug), offset); + if (!callSite) { + return; + } + size_t debugTrapOffset = callSite->returnAddressOffset(); + + const ModuleSegment& codeSegment = code_->segment(Tier::Debug); + const CodeRange* codeRange = + code_->lookupFuncRange(codeSegment.base() + debugTrapOffset); + MOZ_ASSERT(codeRange); + + uint32_t funcIndex = codeRange->funcIndex(); + if (stepperCounters_.lookup(funcIndex)) { + return; // no need to toggle when step mode is enabled + } + + bool anyEnterAndLeave = enterAndLeaveFrameTrapsCounter_ > 0; + bool anyStepping = !stepperCounters_.empty(); + bool anyBreakpoints = !breakpointSites_.empty(); + + if (enabled) { + enableDebuggingForFunction(instance, funcIndex); + enableDebugTrap(instance); + } else if (!anyEnterAndLeave) { + disableDebuggingForFunction(instance, funcIndex); + if (!anyStepping && !anyBreakpoints) { + disableDebugTrap(instance); + } + } +} + +WasmBreakpointSite* DebugState::getBreakpointSite(uint32_t offset) const { + WasmBreakpointSiteMap::Ptr p = breakpointSites_.lookup(offset); + if (!p) { + return nullptr; + } + + return p->value(); +} + +WasmBreakpointSite* DebugState::getOrCreateBreakpointSite(JSContext* cx, + Instance* instance, + uint32_t offset) { + WasmBreakpointSite* site; + + WasmBreakpointSiteMap::AddPtr p = breakpointSites_.lookupForAdd(offset); + if (!p) { + site = cx->new_(instance->object(), offset); + if (!site) { + return nullptr; + } + + if (!breakpointSites_.add(p, offset, site)) { + js_delete(site); + ReportOutOfMemory(cx); + return nullptr; + } + + AddCellMemory(instance->object(), sizeof(WasmBreakpointSite), + MemoryUse::BreakpointSite); + + toggleBreakpointTrap(cx->runtime(), instance, offset, true); + } else { + site = p->value(); + } + return site; +} + +bool DebugState::hasBreakpointSite(uint32_t offset) { + return breakpointSites_.has(offset); +} + +void DebugState::destroyBreakpointSite(JS::GCContext* gcx, Instance* instance, + uint32_t offset) { + WasmBreakpointSiteMap::Ptr p = breakpointSites_.lookup(offset); + MOZ_ASSERT(p); + gcx->delete_(instance->objectUnbarriered(), p->value(), + MemoryUse::BreakpointSite); + breakpointSites_.remove(p); + toggleBreakpointTrap(gcx->runtime(), instance, offset, false); +} + +void DebugState::clearBreakpointsIn(JS::GCContext* gcx, + WasmInstanceObject* instance, + js::Debugger* dbg, JSObject* handler) { + MOZ_ASSERT(instance); + + // Breakpoints hold wrappers in the instance's compartment for the handler. + // Make sure we don't try to search for the unwrapped handler. + MOZ_ASSERT_IF(handler, instance->compartment() == handler->compartment()); + + if (breakpointSites_.empty()) { + return; + } + for (WasmBreakpointSiteMap::Enum e(breakpointSites_); !e.empty(); + e.popFront()) { + WasmBreakpointSite* site = e.front().value(); + MOZ_ASSERT(site->instanceObject == instance); + + Breakpoint* nextbp; + for (Breakpoint* bp = site->firstBreakpoint(); bp; bp = nextbp) { + nextbp = bp->nextInSite(); + MOZ_ASSERT(bp->site == site); + if ((!dbg || bp->debugger == dbg) && + (!handler || bp->getHandler() == handler)) { + bp->delete_(gcx); + } + } + if (site->isEmpty()) { + gcx->delete_(instance, site, MemoryUse::BreakpointSite); + e.removeFront(); + } + } +} + +void DebugState::enableDebuggingForFunction(Instance* instance, + uint32_t funcIndex) { + instance->setDebugFilter(funcIndex, true); +} + +void DebugState::disableDebuggingForFunction(Instance* instance, + uint32_t funcIndex) { + instance->setDebugFilter(funcIndex, false); +} + +void DebugState::enableDebugTrap(Instance* instance) { + instance->setDebugTrapHandler(code_->segment(Tier::Debug).base() + + metadata(Tier::Debug).debugTrapOffset); +} + +void DebugState::disableDebugTrap(Instance* instance) { + instance->setDebugTrapHandler(nullptr); +} + +void DebugState::adjustEnterAndLeaveFrameTrapsState(JSContext* cx, + Instance* instance, + bool enabled) { + MOZ_ASSERT_IF(!enabled, enterAndLeaveFrameTrapsCounter_ > 0); + + bool wasEnabled = enterAndLeaveFrameTrapsCounter_ > 0; + enterAndLeaveFrameTrapsCounter_ += enabled ? 1 : -1; + bool stillEnabled = enterAndLeaveFrameTrapsCounter_ > 0; + if (wasEnabled == stillEnabled) { + return; + } + + MOZ_RELEASE_ASSERT(&instance->metadata() == &metadata()); + uint32_t numFuncs = metadata().debugNumFuncs(); + if (enabled) { + MOZ_ASSERT(enterAndLeaveFrameTrapsCounter_ > 0); + for (uint32_t funcIdx = 0; funcIdx < numFuncs; funcIdx++) { + enableDebuggingForFunction(instance, funcIdx); + } + enableDebugTrap(instance); + } else { + MOZ_ASSERT(enterAndLeaveFrameTrapsCounter_ == 0); + bool anyEnabled = false; + for (uint32_t funcIdx = 0; funcIdx < numFuncs; funcIdx++) { + // For each function, disable the bit if nothing else is going on. This + // means determining if there's stepping or breakpoints. + bool mustLeaveEnabled = stepperCounters_.lookup(funcIdx).found(); + for (auto iter = breakpointSites_.iter(); + !iter.done() && !mustLeaveEnabled; iter.next()) { + WasmBreakpointSite* site = iter.get().value(); + const CallSite* callSite = + SlowCallSiteSearchByOffset(metadata(Tier::Debug), site->offset); + if (callSite) { + size_t debugTrapOffset = callSite->returnAddressOffset(); + const ModuleSegment& codeSegment = code_->segment(Tier::Debug); + const CodeRange* codeRange = + code_->lookupFuncRange(codeSegment.base() + debugTrapOffset); + MOZ_ASSERT(codeRange); + mustLeaveEnabled = codeRange->funcIndex() == funcIdx; + } + } + if (mustLeaveEnabled) { + anyEnabled = true; + } else { + disableDebuggingForFunction(instance, funcIdx); + } + } + if (!anyEnabled) { + disableDebugTrap(instance); + } + } +} + +void DebugState::ensureEnterFrameTrapsState(JSContext* cx, Instance* instance, + bool enabled) { + if (enterFrameTrapsEnabled_ == enabled) { + return; + } + + adjustEnterAndLeaveFrameTrapsState(cx, instance, enabled); + + enterFrameTrapsEnabled_ = enabled; +} + +bool DebugState::debugGetLocalTypes(uint32_t funcIndex, ValTypeVector* locals, + size_t* argsLength, + StackResults* stackResults) { + const TypeContext& types = *metadata().types; + const FuncType& funcType = metadata().debugFuncType(funcIndex); + const ValTypeVector& args = funcType.args(); + const ValTypeVector& results = funcType.results(); + ResultType resultType(ResultType::Vector(results)); + *argsLength = args.length(); + *stackResults = ABIResultIter::HasStackResults(resultType) + ? StackResults::HasStackResults + : StackResults::NoStackResults; + if (!locals->appendAll(args)) { + return false; + } + + // Decode local var types from wasm binary function body. + const CodeRange& range = + codeRanges(Tier::Debug)[funcToCodeRangeIndex(funcIndex)]; + // In wasm, the Code points to the function start via funcLineOrBytecode. + size_t offsetInModule = range.funcLineOrBytecode(); + Decoder d(bytecode().begin() + offsetInModule, bytecode().end(), + offsetInModule, + /* error = */ nullptr); + return DecodeValidatedLocalEntries(types, d, locals); +} + +bool DebugState::getGlobal(Instance& instance, uint32_t globalIndex, + MutableHandleValue vp) { + const GlobalDesc& global = metadata().globals[globalIndex]; + + if (global.isConstant()) { + LitVal value = global.constantValue(); + switch (value.type().kind()) { + case ValType::I32: + vp.set(Int32Value(value.i32())); + break; + case ValType::I64: + // Just display as a Number; it's ok if we lose some precision + vp.set(NumberValue((double)value.i64())); + break; + case ValType::F32: + vp.set(NumberValue(JS::CanonicalizeNaN(value.f32()))); + break; + case ValType::F64: + vp.set(NumberValue(JS::CanonicalizeNaN(value.f64()))); + break; + case ValType::Ref: + // It's possible to do better. We could try some kind of hashing + // scheme, to make the pointer recognizable without revealing it. + vp.set(MagicValue(JS_OPTIMIZED_OUT)); + break; + case ValType::V128: + // Debugger must be updated to handle this, and should be updated to + // handle i64 in any case. + vp.set(MagicValue(JS_OPTIMIZED_OUT)); + break; + default: + MOZ_CRASH("Global constant type"); + } + return true; + } + + void* dataPtr = instance.data() + global.offset(); + if (global.isIndirect()) { + dataPtr = *static_cast(dataPtr); + } + switch (global.type().kind()) { + case ValType::I32: { + vp.set(Int32Value(*static_cast(dataPtr))); + break; + } + case ValType::I64: { + // Just display as a Number; it's ok if we lose some precision + vp.set(NumberValue((double)*static_cast(dataPtr))); + break; + } + case ValType::F32: { + vp.set(NumberValue(JS::CanonicalizeNaN(*static_cast(dataPtr)))); + break; + } + case ValType::F64: { + vp.set(NumberValue(JS::CanonicalizeNaN(*static_cast(dataPtr)))); + break; + } + case ValType::Ref: { + // Just hide it. See above. + vp.set(MagicValue(JS_OPTIMIZED_OUT)); + break; + } + case ValType::V128: { + // Just hide it. See above. + vp.set(MagicValue(JS_OPTIMIZED_OUT)); + break; + } + default: { + MOZ_CRASH("Global variable type"); + break; + } + } + return true; +} + +bool DebugState::getSourceMappingURL(JSContext* cx, + MutableHandleString result) const { + result.set(nullptr); + + for (const CustomSection& customSection : module_->customSections()) { + const Bytes& sectionName = customSection.name; + if (strlen(SourceMappingURLSectionName) != sectionName.length() || + memcmp(SourceMappingURLSectionName, sectionName.begin(), + sectionName.length()) != 0) { + continue; + } + + // Parse found "SourceMappingURL" custom section. + Decoder d(customSection.payload->begin(), customSection.payload->end(), 0, + /* error = */ nullptr); + uint32_t nchars; + if (!d.readVarU32(&nchars)) { + return true; // ignoring invalid section data + } + const uint8_t* chars; + if (!d.readBytes(nchars, &chars) || d.currentPosition() != d.end()) { + return true; // ignoring invalid section data + } + + JS::UTF8Chars utf8Chars(reinterpret_cast(chars), nchars); + JSString* str = JS_NewStringCopyUTF8N(cx, utf8Chars); + if (!str) { + return false; + } + result.set(str); + return true; + } + + // Check presence of "SourceMap:" HTTP response header. + char* sourceMapURL = metadata().sourceMapURL.get(); + if (sourceMapURL && strlen(sourceMapURL)) { + JS::UTF8Chars utf8Chars(sourceMapURL, strlen(sourceMapURL)); + JSString* str = JS_NewStringCopyUTF8N(cx, utf8Chars); + if (!str) { + return false; + } + result.set(str); + } + return true; +} + +void DebugState::addSizeOfMisc(MallocSizeOf mallocSizeOf, + Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, size_t* code, + size_t* data) const { + code_->addSizeOfMiscIfNotSeen(mallocSizeOf, seenMetadata, seenCode, code, + data); + module_->addSizeOfMisc(mallocSizeOf, seenMetadata, seenCode, code, data); +} diff --git a/js/src/wasm/WasmDebug.h b/js/src/wasm/WasmDebug.h new file mode 100644 index 0000000000..35c8a838ed --- /dev/null +++ b/js/src/wasm/WasmDebug.h @@ -0,0 +1,187 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_debug_h +#define wasm_debug_h + +#include "js/HashTable.h" +#include "wasm/WasmCode.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmExprType.h" +#include "wasm/WasmModule.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmValType.h" + +namespace js { + +class Debugger; +class WasmBreakpointSite; +class WasmInstanceObject; + +namespace wasm { + +struct MetadataTier; + +// The generated source location for the AST node/expression. The offset field +// refers an offset in an binary format file. + +struct ExprLoc { + uint32_t lineno; + uint32_t column; + uint32_t offset; + ExprLoc() : lineno(0), column(0), offset(0) {} + ExprLoc(uint32_t lineno_, uint32_t column_, uint32_t offset_) + : lineno(lineno_), column(column_), offset(offset_) {} +}; + +using StepperCounters = + HashMap, SystemAllocPolicy>; +using WasmBreakpointSiteMap = + HashMap, + SystemAllocPolicy>; + +/* + * [SMDOC] Wasm debug traps + * + * There is a single debug trap handler for the process, WasmHandleDebugTrap in + * WasmBuiltins.cpp. That function is invoked through the Debug Trap Stub, + * generated by GenerateDebugTrapStub in WasmStubs.cpp. When any function in an + * instance needs to trap for any reason (enter frame, leave frame, breakpoint, + * or single-stepping) then a pointer to the Debug Trap Stub is installed in the + * Instance. Debug-enabled code will look for this pointer and call it if it is + * not null. + * + * WasmHandleDebugTrap may therefore be called very frequently when any function + * in the instance is being debugged, and must filter the trap against the + * tables in the DebugState. It can make use of the return address for the + * call, which identifies the site uniquely. + * + * In order to greatly reduce the frequency of calls to the Debug Trap Stub, an + * array of flag bits, one per function, is attached to the instance. The code + * at the breakable point calls a shared stub within the function containing the + * breakable point to check whether the bit is set for the function. If it is + * not set, the stub can return to its caller immediately; if the bit is set, + * the stub will jump to the installed Debug Trap Stub. + */ + +class DebugState { + const SharedCode code_; + const SharedModule module_; + + // State maintained when debugging is enabled. + + bool enterFrameTrapsEnabled_; + uint32_t enterAndLeaveFrameTrapsCounter_; + WasmBreakpointSiteMap breakpointSites_; + StepperCounters stepperCounters_; + + void enableDebuggingForFunction(Instance* instance, uint32_t funcIndex); + void disableDebuggingForFunction(Instance* instance, uint32_t funcIndex); + void enableDebugTrap(Instance* instance); + void disableDebugTrap(Instance* instance); + + public: + DebugState(const Code& code, const Module& module); + + void trace(JSTracer* trc); + void finalize(JS::GCContext* gcx); + + const Bytes& bytecode() const { return module_->debugBytecode(); } + + [[nodiscard]] bool getLineOffsets(size_t lineno, Vector* offsets); + [[nodiscard]] bool getAllColumnOffsets(Vector* offsets); + [[nodiscard]] bool getOffsetLocation(uint32_t offset, size_t* lineno, + size_t* column); + + // The Code can track enter/leave frame events. Any such event triggers + // debug trap. The enter/leave frame events enabled or disabled across + // all functions. + + void adjustEnterAndLeaveFrameTrapsState(JSContext* cx, Instance* instance, + bool enabled); + void ensureEnterFrameTrapsState(JSContext* cx, Instance* instance, + bool enabled); + bool enterFrameTrapsEnabled() const { return enterFrameTrapsEnabled_; } + + // When the Code is debugEnabled, individual breakpoints can be enabled or + // disabled at instruction offsets. + + bool hasBreakpointTrapAtOffset(uint32_t offset); + void toggleBreakpointTrap(JSRuntime* rt, Instance* instance, uint32_t offset, + bool enabled); + WasmBreakpointSite* getBreakpointSite(uint32_t offset) const; + WasmBreakpointSite* getOrCreateBreakpointSite(JSContext* cx, + Instance* instance, + uint32_t offset); + bool hasBreakpointSite(uint32_t offset); + void destroyBreakpointSite(JS::GCContext* gcx, Instance* instance, + uint32_t offset); + void clearBreakpointsIn(JS::GCContext* gcx, WasmInstanceObject* instance, + js::Debugger* dbg, JSObject* handler); + + // When the Code is debug-enabled, single-stepping mode can be toggled on + // the granularity of individual functions. + + bool stepModeEnabled(uint32_t funcIndex) const; + [[nodiscard]] bool incrementStepperCount(JSContext* cx, Instance* instance, + uint32_t funcIndex); + void decrementStepperCount(JS::GCContext* gcx, Instance* instance, + uint32_t funcIndex); + + // Stack inspection helpers. + + [[nodiscard]] bool debugGetLocalTypes(uint32_t funcIndex, + ValTypeVector* locals, + size_t* argsLength, + StackResults* stackResults); + [[nodiscard]] bool getGlobal(Instance& instance, uint32_t globalIndex, + MutableHandleValue vp); + + // Debug URL helpers. + + [[nodiscard]] bool getSourceMappingURL(JSContext* cx, + MutableHandleString result) const; + + // Accessors for commonly used elements of linked structures. + + const MetadataTier& metadata(Tier t) const { return code_->metadata(t); } + const Metadata& metadata() const { return code_->metadata(); } + const CodeRangeVector& codeRanges(Tier t) const { + return metadata(t).codeRanges; + } + const CallSiteVector& callSites(Tier t) const { + return metadata(t).callSites; + } + + uint32_t funcToCodeRangeIndex(uint32_t funcIndex) const { + return metadata(Tier::Debug).funcToCodeRange[funcIndex]; + } + + // about:memory reporting: + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, size_t* code, size_t* data) const; +}; + +using UniqueDebugState = UniquePtr; + +} // namespace wasm +} // namespace js + +#endif // wasm_debug_h diff --git a/js/src/wasm/WasmDebugFrame.cpp b/js/src/wasm/WasmDebugFrame.cpp new file mode 100644 index 0000000000..2c93e750bc --- /dev/null +++ b/js/src/wasm/WasmDebugFrame.cpp @@ -0,0 +1,176 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmDebugFrame.h" + +#include "vm/EnvironmentObject.h" +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmDebug.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmInstanceData.h" +#include "wasm/WasmStubs.h" + +#include "vm/NativeObject-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +/* static */ +DebugFrame* DebugFrame::from(Frame* fp) { + MOZ_ASSERT(GetNearestEffectiveInstance(fp)->code().metadata().debugEnabled); + auto* df = + reinterpret_cast((uint8_t*)fp - DebugFrame::offsetOfFrame()); + MOZ_ASSERT(GetNearestEffectiveInstance(fp) == df->instance()); + return df; +} + +void DebugFrame::alignmentStaticAsserts() { + // VS2017 doesn't consider offsetOfFrame() to be a constexpr, so we have + // to use offsetof directly. These asserts can't be at class-level + // because the type is incomplete. + + static_assert(WasmStackAlignment >= Alignment, + "Aligned by ABI before pushing DebugFrame"); +#ifndef JS_CODEGEN_NONE + static_assert((offsetof(DebugFrame, frame_) + sizeof(Frame)) % Alignment == 0, + "Aligned after pushing DebugFrame"); +#endif +#ifdef JS_CODEGEN_ARM64 + // This constraint may or may not be necessary. If you hit this because + // you've changed the frame size then feel free to remove it, but be extra + // aware of possible problems. + static_assert(sizeof(DebugFrame) % 16 == 0, "ARM64 SP alignment"); +#endif +} + +Instance* DebugFrame::instance() { + return GetNearestEffectiveInstance(&frame_); +} + +const Instance* DebugFrame::instance() const { + return GetNearestEffectiveInstance(&frame_); +} + +GlobalObject* DebugFrame::global() { return &instance()->object()->global(); } + +bool DebugFrame::hasGlobal(const GlobalObject* global) const { + return global == &instance()->objectUnbarriered()->global(); +} + +JSObject* DebugFrame::environmentChain() { + return &global()->lexicalEnvironment(); +} + +bool DebugFrame::getLocal(uint32_t localIndex, MutableHandleValue vp) { + ValTypeVector locals; + size_t argsLength; + StackResults stackResults; + if (!instance()->debug().debugGetLocalTypes(funcIndex(), &locals, &argsLength, + &stackResults)) { + return false; + } + + ValTypeVector args; + MOZ_ASSERT(argsLength <= locals.length()); + if (!args.append(locals.begin(), argsLength)) { + return false; + } + ArgTypeVector abiArgs(args, stackResults); + + BaseLocalIter iter(locals, abiArgs, /* debugEnabled = */ true); + while (!iter.done() && iter.index() < localIndex) { + iter++; + } + MOZ_ALWAYS_TRUE(!iter.done()); + + uint8_t* frame = static_cast((void*)this) + offsetOfFrame(); + void* dataPtr = frame - iter.frameOffset(); + switch (iter.mirType()) { + case jit::MIRType::Int32: + vp.set(Int32Value(*static_cast(dataPtr))); + break; + case jit::MIRType::Int64: + // Just display as a Number; it's ok if we lose some precision + vp.set(NumberValue((double)*static_cast(dataPtr))); + break; + case jit::MIRType::Float32: + vp.set(NumberValue(JS::CanonicalizeNaN(*static_cast(dataPtr)))); + break; + case jit::MIRType::Double: + vp.set(NumberValue(JS::CanonicalizeNaN(*static_cast(dataPtr)))); + break; + case jit::MIRType::RefOrNull: + vp.set(ObjectOrNullValue(*(JSObject**)dataPtr)); + break; +#ifdef ENABLE_WASM_SIMD + case jit::MIRType::Simd128: + vp.set(NumberValue(0)); + break; +#endif + default: + MOZ_CRASH("local type"); + } + return true; +} + +bool DebugFrame::updateReturnJSValue(JSContext* cx) { + MutableHandleValue rval = + MutableHandleValue::fromMarkedLocation(&cachedReturnJSValue_); + rval.setUndefined(); + flags_.hasCachedReturnJSValue = true; + ResultType resultType = ResultType::Vector( + instance()->metadata().debugFuncType(funcIndex()).results()); + Maybe stackResultsLoc; + if (ABIResultIter::HasStackResults(resultType)) { + stackResultsLoc = Some(static_cast(stackResultsPointer_)); + } + DebugCodegen(DebugChannel::Function, + "wasm-function[%d] updateReturnJSValue [", funcIndex()); + bool ok = + ResultsToJSValue(cx, resultType, registerResults_, stackResultsLoc, rval); + DebugCodegen(DebugChannel::Function, "]\n"); + return ok; +} + +HandleValue DebugFrame::returnValue() const { + MOZ_ASSERT(flags_.hasCachedReturnJSValue); + return HandleValue::fromMarkedLocation(&cachedReturnJSValue_); +} + +void DebugFrame::clearReturnJSValue() { + flags_.hasCachedReturnJSValue = true; + cachedReturnJSValue_.setUndefined(); +} + +void DebugFrame::observe(JSContext* cx) { + if (!flags_.observing) { + instance()->debug().adjustEnterAndLeaveFrameTrapsState( + cx, instance(), /* enabled = */ true); + flags_.observing = true; + } +} + +void DebugFrame::leave(JSContext* cx) { + if (flags_.observing) { + instance()->debug().adjustEnterAndLeaveFrameTrapsState( + cx, instance(), /* enabled = */ false); + flags_.observing = false; + } +} diff --git a/js/src/wasm/WasmDebugFrame.h b/js/src/wasm/WasmDebugFrame.h new file mode 100644 index 0000000000..5a9ec6c783 --- /dev/null +++ b/js/src/wasm/WasmDebugFrame.h @@ -0,0 +1,217 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_debugframe_h +#define wasm_debugframe_h + +#include "mozilla/Assertions.h" + +#include +#include + +#include "js/TypeDecls.h" +#include "js/Value.h" +#include "wasm/WasmCodegenConstants.h" +#include "wasm/WasmFrame.h" +#include "wasm/WasmValType.h" +#include "wasm/WasmValue.h" + +namespace js { + +class GlobalObject; + +namespace wasm { + +class Instance; + +// A DebugFrame is a Frame with additional fields that are added after the +// normal function prologue by the baseline compiler. If a Module is compiled +// with debugging enabled, then all its code creates DebugFrames on the stack +// instead of just Frames. These extra fields are used by the Debugger API. + +class DebugFrame { + // The register results field. Initialized only during the baseline + // compiler's return sequence to allow the debugger to inspect and + // modify the return values of a frame being debugged. + union SpilledRegisterResult { + private: + int32_t i32_; + int64_t i64_; + float f32_; + double f64_; +#ifdef ENABLE_WASM_SIMD + V128 v128_; +#endif + AnyRef anyref_; + +#ifdef DEBUG + // Should we add a new value representation, this will remind us to update + // SpilledRegisterResult. + static inline void assertAllValueTypesHandled(ValType type) { + switch (type.kind()) { + case ValType::I32: + case ValType::I64: + case ValType::F32: + case ValType::F64: + case ValType::V128: + case ValType::Ref: + return; + } + } +#endif + }; + SpilledRegisterResult registerResults_[MaxRegisterResults]; + + // The returnValue() method returns a HandleValue pointing to this field. + JS::Value cachedReturnJSValue_; + + // If the function returns multiple results, this field is initialized + // to a pointer to the stack results. + void* stackResultsPointer_; + + // The function index of this frame. Technically, this could be derived + // given a PC into this frame (which could lookup the CodeRange which has + // the function index), but this isn't always readily available. + uint32_t funcIndex_; + + // Flags whose meaning are described below. + union Flags { + struct { + uint32_t observing : 1; + uint32_t isDebuggee : 1; + uint32_t prevUpToDate : 1; + uint32_t hasCachedSavedFrame : 1; + uint32_t hasCachedReturnJSValue : 1; + uint32_t hasSpilledRefRegisterResult : MaxRegisterResults; + }; + uint32_t allFlags; + } flags_; + + // Avoid -Wunused-private-field warnings. + protected: +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_X86) || defined(__wasi__) + // See alignmentStaticAsserts(). For ARM32 and X86 DebugFrame is only + // 4-byte aligned, so we add another word to get up to 8-byte + // alignment. + uint32_t padding_; +#endif +#if defined(ENABLE_WASM_SIMD) && defined(JS_CODEGEN_ARM64) + uint64_t padding_; +#endif + + private: + // The Frame goes at the end since the stack grows down. + Frame frame_; + + public: + static DebugFrame* from(Frame* fp); + Frame& frame() { return frame_; } + uint32_t funcIndex() const { return funcIndex_; } + Instance* instance(); + const Instance* instance() const; + GlobalObject* global(); + bool hasGlobal(const GlobalObject* global) const; + JSObject* environmentChain(); + bool getLocal(uint32_t localIndex, JS::MutableHandleValue vp); + + // The return value must be written from the unboxed representation in the + // results union into cachedReturnJSValue_ by updateReturnJSValue() before + // returnValue() can return a Handle to it. + + bool hasCachedReturnJSValue() const { return flags_.hasCachedReturnJSValue; } + [[nodiscard]] bool updateReturnJSValue(JSContext* cx); + JS::HandleValue returnValue() const; + void clearReturnJSValue(); + + // Once the debugger observes a frame, it must be notified via + // onLeaveFrame() before the frame is popped. Calling observe() ensures the + // leave frame traps are enabled. Both methods are idempotent so the caller + // doesn't have to worry about calling them more than once. + + void observe(JSContext* cx); + void leave(JSContext* cx); + + // The 'isDebugge' bit is initialized to false and set by the WebAssembly + // runtime right before a frame is exposed to the debugger, as required by + // the Debugger API. The bit is then used for Debugger-internal purposes + // afterwards. + + bool isDebuggee() const { return flags_.isDebuggee; } + void setIsDebuggee() { flags_.isDebuggee = true; } + void unsetIsDebuggee() { flags_.isDebuggee = false; } + + // These are opaque boolean flags used by the debugger to implement + // AbstractFramePtr. They are initialized to false and not otherwise read or + // written by wasm code or runtime. + + bool prevUpToDate() const { return flags_.prevUpToDate; } + void setPrevUpToDate() { flags_.prevUpToDate = true; } + void unsetPrevUpToDate() { flags_.prevUpToDate = false; } + + bool hasCachedSavedFrame() const { return flags_.hasCachedSavedFrame; } + void setHasCachedSavedFrame() { flags_.hasCachedSavedFrame = true; } + void clearHasCachedSavedFrame() { flags_.hasCachedSavedFrame = false; } + + bool hasSpilledRegisterRefResult(size_t n) const { + uint32_t mask = hasSpilledRegisterRefResultBitMask(n); + return (flags_.allFlags & mask) != 0; + } + + // DebugFrame is accessed directly by JIT code. + + static constexpr size_t offsetOfRegisterResults() { + return offsetof(DebugFrame, registerResults_); + } + static constexpr size_t offsetOfRegisterResult(size_t n) { + MOZ_ASSERT(n < MaxRegisterResults); + return offsetOfRegisterResults() + n * sizeof(SpilledRegisterResult); + } + static constexpr size_t offsetOfCachedReturnJSValue() { + return offsetof(DebugFrame, cachedReturnJSValue_); + } + static constexpr size_t offsetOfStackResultsPointer() { + return offsetof(DebugFrame, stackResultsPointer_); + } + static constexpr size_t offsetOfFlags() { + return offsetof(DebugFrame, flags_); + } + static constexpr uint32_t hasSpilledRegisterRefResultBitMask(size_t n) { + MOZ_ASSERT(n < MaxRegisterResults); + union Flags flags = {.allFlags = 0}; + flags.hasSpilledRefRegisterResult = 1 << n; + MOZ_ASSERT(flags.allFlags != 0); + return flags.allFlags; + } + static constexpr size_t offsetOfFuncIndex() { + return offsetof(DebugFrame, funcIndex_); + } + static constexpr size_t offsetOfFrame() { + return offsetof(DebugFrame, frame_); + } + + // DebugFrames are aligned to 8-byte aligned, allowing them to be placed in + // an AbstractFramePtr. + + static const unsigned Alignment = 8; + static void alignmentStaticAsserts(); +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_debugframe_h diff --git a/js/src/wasm/WasmException.h b/js/src/wasm/WasmException.h new file mode 100644 index 0000000000..d9cfcfb8b0 --- /dev/null +++ b/js/src/wasm/WasmException.h @@ -0,0 +1,31 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_exception_h +#define wasm_exception_h + +namespace js { +namespace wasm { + +static const uint32_t CatchAllIndex = UINT32_MAX; +static_assert(CatchAllIndex > MaxTags); + +} // namespace wasm +} // namespace js + +#endif // wasm_exception_h diff --git a/js/src/wasm/WasmExprType.h b/js/src/wasm/WasmExprType.h new file mode 100644 index 0000000000..4b02cd5e60 --- /dev/null +++ b/js/src/wasm/WasmExprType.h @@ -0,0 +1,330 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_expr_type_h +#define wasm_expr_type_h + +#include + +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValType.h" + +namespace js { +namespace wasm { + +template +class TaggedValue { + public: + enum Kind { + ImmediateKind1 = 0, + ImmediateKind2 = 1, + PointerKind1 = 2, + PointerKind2 = 3 + }; + using PackedRepr = uint64_t; + static_assert(std::is_same(), + "can use pointer tagging with PackedTypeCode"); + + private: + PackedRepr bits_; + + static constexpr PackedRepr PayloadShift = 2; + static constexpr PackedRepr KindMask = 0x3; + static constexpr PackedRepr PointerKindBit = 0x2; + + constexpr static bool IsPointerKind(Kind kind) { + return PackedRepr(kind) & PointerKindBit; + } + constexpr static bool IsImmediateKind(Kind kind) { + return !IsPointerKind(kind); + } + + static_assert(IsImmediateKind(ImmediateKind1), "immediate kind 1"); + static_assert(IsImmediateKind(ImmediateKind2), "immediate kind 2"); + static_assert(IsPointerKind(PointerKind1), "pointer kind 1"); + static_assert(IsPointerKind(PointerKind2), "pointer kind 2"); + + static PackedRepr PackImmediate(Kind kind, PackedRepr imm) { + MOZ_ASSERT(IsImmediateKind(kind)); + MOZ_ASSERT((PackedRepr(kind) & KindMask) == kind); + MOZ_ASSERT((imm & (PackedRepr(KindMask) + << ((sizeof(PackedRepr) * 8) - PayloadShift))) == 0); + return PackedRepr(kind) | (PackedRepr(imm) << PayloadShift); + } + + static PackedRepr PackPointer(Kind kind, PointerType* ptr) { + PackedRepr ptrBits = reinterpret_cast(ptr); + MOZ_ASSERT(IsPointerKind(kind)); + MOZ_ASSERT((PackedRepr(kind) & KindMask) == kind); + MOZ_ASSERT((ptrBits & KindMask) == 0); + return PackedRepr(kind) | ptrBits; + } + + public: + TaggedValue(Kind kind, PackedRepr imm) : bits_(PackImmediate(kind, imm)) {} + TaggedValue(Kind kind, PointerType* ptr) : bits_(PackPointer(kind, ptr)) {} + + PackedRepr bits() const { return bits_; } + Kind kind() const { return Kind(bits() & KindMask); } + PackedRepr immediate() const { + MOZ_ASSERT(IsImmediateKind(kind())); + return mozilla::AssertedCast(bits() >> PayloadShift); + } + PointerType* pointer() const { + MOZ_ASSERT(IsPointerKind(kind())); + return reinterpret_cast(bits() & ~KindMask); + } +}; + +// ResultType represents the WebAssembly spec's `resulttype`. Semantically, a +// result type is just a vec(valtype). For effiency, though, the ResultType +// value is packed into a word, with separate encodings for these 3 cases: +// [] +// [valtype] +// pointer to ValTypeVector +// +// Additionally there is an encoding indicating uninitialized ResultType +// values. +// +// Generally in the latter case the ValTypeVector is the args() or results() of +// a FuncType in the compilation unit, so as long as the lifetime of the +// ResultType value is less than the OpIter, we can just borrow the pointer +// without ownership or copying. +class ResultType { + using Tagged = TaggedValue; + Tagged tagged_; + + enum Kind { + EmptyKind = Tagged::ImmediateKind1, + SingleKind = Tagged::ImmediateKind2, + VectorKind = Tagged::PointerKind1, + InvalidKind = Tagged::PointerKind2, + }; + + ResultType(Kind kind, Tagged::PackedRepr imm) + : tagged_(Tagged::Kind(kind), imm) {} + explicit ResultType(const ValTypeVector* ptr) + : tagged_(Tagged::Kind(VectorKind), ptr) {} + + Kind kind() const { return Kind(tagged_.kind()); } + + ValType singleValType() const { + MOZ_ASSERT(kind() == SingleKind); + return ValType(PackedTypeCode::fromBits(tagged_.immediate())); + } + + const ValTypeVector& values() const { + MOZ_ASSERT(kind() == VectorKind); + return *tagged_.pointer(); + } + + public: + ResultType() : tagged_(Tagged::Kind(InvalidKind), nullptr) {} + + static ResultType Empty() { + return ResultType(EmptyKind, Tagged::PackedRepr(0)); + } + static ResultType Single(ValType vt) { + return ResultType(SingleKind, vt.bitsUnsafe()); + } + static ResultType Vector(const ValTypeVector& vals) { + switch (vals.length()) { + case 0: + return Empty(); + case 1: + return Single(vals[0]); + default: + return ResultType(&vals); + } + } + + [[nodiscard]] bool cloneToVector(ValTypeVector* out) { + MOZ_ASSERT(out->empty()); + switch (kind()) { + case EmptyKind: + return true; + case SingleKind: + return out->append(singleValType()); + case VectorKind: + return out->appendAll(values()); + default: + MOZ_CRASH("bad resulttype"); + } + } + + bool valid() const { return kind() != InvalidKind; } + bool empty() const { return kind() == EmptyKind; } + + size_t length() const { + switch (kind()) { + case EmptyKind: + return 0; + case SingleKind: + return 1; + case VectorKind: + return values().length(); + default: + MOZ_CRASH("bad resulttype"); + } + } + + // Polyfill the Span API, which is polyfilling the std library + size_t size() const { return length(); } + + ValType operator[](size_t i) const { + switch (kind()) { + case SingleKind: + MOZ_ASSERT(i == 0); + return singleValType(); + case VectorKind: + return values()[i]; + default: + MOZ_CRASH("bad resulttype"); + } + } + + bool operator==(ResultType rhs) const { + switch (kind()) { + case EmptyKind: + case SingleKind: + case InvalidKind: + return tagged_.bits() == rhs.tagged_.bits(); + case VectorKind: { + if (rhs.kind() != VectorKind) { + return false; + } + return EqualContainers(values(), rhs.values()); + } + default: + MOZ_CRASH("bad resulttype"); + } + } + bool operator!=(ResultType rhs) const { return !(*this == rhs); } +}; + +// BlockType represents the WebAssembly spec's `blocktype`. Semantically, a +// block type is just a (vec(valtype) -> vec(valtype)) with four special +// encodings which are represented explicitly in BlockType: +// [] -> [] +// [] -> [valtype] +// [params] -> [results] via pointer to FuncType +// [] -> [results] via pointer to FuncType (ignoring [params]) + +class BlockType { + using Tagged = TaggedValue; + Tagged tagged_; + + enum Kind { + VoidToVoidKind = Tagged::ImmediateKind1, + VoidToSingleKind = Tagged::ImmediateKind2, + FuncKind = Tagged::PointerKind1, + FuncResultsKind = Tagged::PointerKind2 + }; + + BlockType(Kind kind, Tagged::PackedRepr imm) + : tagged_(Tagged::Kind(kind), imm) {} + BlockType(Kind kind, const FuncType& type) + : tagged_(Tagged::Kind(kind), &type) {} + + Kind kind() const { return Kind(tagged_.kind()); } + ValType singleValType() const { + MOZ_ASSERT(kind() == VoidToSingleKind); + return ValType(PackedTypeCode::fromBits(tagged_.immediate())); + } + + const FuncType& funcType() const { return *tagged_.pointer(); } + + public: + BlockType() + : tagged_(Tagged::Kind(VoidToVoidKind), + PackedTypeCode::invalid().bits()) {} + + static BlockType VoidToVoid() { + return BlockType(VoidToVoidKind, Tagged::PackedRepr(0)); + } + static BlockType VoidToSingle(ValType vt) { + return BlockType(VoidToSingleKind, vt.bitsUnsafe()); + } + static BlockType Func(const FuncType& type) { + if (type.args().length() == 0) { + return FuncResults(type); + } + return BlockType(FuncKind, type); + } + static BlockType FuncResults(const FuncType& type) { + switch (type.results().length()) { + case 0: + return VoidToVoid(); + case 1: + return VoidToSingle(type.results()[0]); + default: + return BlockType(FuncResultsKind, type); + } + } + + ResultType params() const { + switch (kind()) { + case VoidToVoidKind: + case VoidToSingleKind: + case FuncResultsKind: + return ResultType::Empty(); + case FuncKind: + return ResultType::Vector(funcType().args()); + default: + MOZ_CRASH("unexpected kind"); + } + } + + ResultType results() const { + switch (kind()) { + case VoidToVoidKind: + return ResultType::Empty(); + case VoidToSingleKind: + return ResultType::Single(singleValType()); + case FuncKind: + case FuncResultsKind: + return ResultType::Vector(funcType().results()); + default: + MOZ_CRASH("unexpected kind"); + } + } + + bool operator==(BlockType rhs) const { + if (kind() != rhs.kind()) { + return false; + } + switch (kind()) { + case VoidToVoidKind: + case VoidToSingleKind: + return tagged_.bits() == rhs.tagged_.bits(); + case FuncKind: + return FuncType::strictlyEquals(funcType(), rhs.funcType()); + case FuncResultsKind: + return EqualContainers(funcType().results(), rhs.funcType().results()); + default: + MOZ_CRASH("unexpected kind"); + } + } + + bool operator!=(BlockType rhs) const { return !(*this == rhs); } +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_expr_type_h diff --git a/js/src/wasm/WasmFrame.h b/js/src/wasm/WasmFrame.h new file mode 100644 index 0000000000..19bebae9ca --- /dev/null +++ b/js/src/wasm/WasmFrame.h @@ -0,0 +1,409 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* [SMDOC] The WASM ABIs + * + * Wasm-internal ABI. + * + * The *Wasm-internal ABI* is the ABI a wasm function assumes when it is + * entered, and the one it assumes when it is making a call to what it believes + * is another wasm function. + * + * We pass the first function arguments in registers (GPR and FPU both) and the + * rest on the stack, generally according to platform ABI conventions (which can + * be hairy). On x86-32 there are no register arguments. + * + * We have no callee-saves registers in the wasm-internal ABI, regardless of the + * platform ABI conventions, though see below about InstanceReg or HeapReg. + * + * We return the last return value in the first return register, according to + * platform ABI conventions. If there is more than one return value, an area is + * allocated in the caller's frame to receive the other return values, and the + * address of this area is passed to the callee as the last argument. Return + * values except the last are stored in ascending order within this area. Also + * see below about alignment of this area and the values in it. + * + * When a function is entered, there are two incoming register values in + * addition to the function's declared parameters: InstanceReg must have the + * correct instance pointer, and HeapReg the correct memoryBase, for the + * function. (On x86-32 there is no HeapReg.) From the instance we can get to + * the JSContext, the instance, the MemoryBase, and many other things. The + * instance maps one-to-one with an instance. + * + * HeapReg and InstanceReg are not parameters in the usual sense, nor are they + * callee-saves registers. Instead they constitute global register state, the + * purpose of which is to bias the call ABI in favor of intra-instance calls, + * the predominant case where the caller and the callee have the same + * InstanceReg and HeapReg values. + * + * With this global register state, literally no work needs to take place to + * save and restore the instance and MemoryBase values across intra-instance + * call boundaries. + * + * For inter-instance calls, in contrast, there must be an instance switch at + * the call boundary: Before the call, the callee's instance must be loaded + * (from a closure or from the import table), and from the instance we load the + * callee's MemoryBase, the realm, and the JSContext. The caller's and callee's + * instance values must be stored into the frame (to aid unwinding), the + * callee's realm must be stored into the JSContext, and the callee's instance + * and MemoryBase values must be moved to appropriate registers. After the + * call, the caller's instance must be loaded, and from it the caller's + * MemoryBase and realm, and the JSContext. The realm must be stored into the + * JSContext and the caller's instance and MemoryBase values must be moved to + * appropriate registers. + * + * Direct calls to functions within the same module are always intra-instance, + * while direct calls to imported functions are always inter-instance. Indirect + * calls -- call_indirect in the MVP, future call_ref and call_funcref -- may or + * may not be intra-instance. + * + * call_indirect, and future call_funcref, also pass a signature value in a + * register (even on x86-32), this is a small integer or a pointer value + * denoting the caller's expected function signature. The callee must compare + * it to the value or pointer that denotes its actual signature, and trap on + * mismatch. + * + * This is what the stack looks like during a call, after the callee has + * completed the prologue: + * + * | | + * +-----------------------------------+ <-+ + * | ... | | + * | Caller's private frame | | + * +-----------------------------------+ | + * | Multi-value return (optional) | | + * | ... | | + * +-----------------------------------+ | + * | Stack args (optional) | | + * | ... | | + * +-----------------------------------+ -+| + * | Caller instance slot | \ + * | Callee instance slot | | \ + * +-----------------------------------+ | \ + * | Shadowstack area (Win64) | | wasm::FrameWithInstances + * | (32 bytes) | | / + * +-----------------------------------+ | / <= SP "Before call" + * | Return address | // <= SP "After call" + * | Saved FP ----|--+/ + * +-----------------------------------+ -+ <= FP (a wasm::Frame*) + * | DebugFrame, Locals, spills, etc | + * | (i.e., callee's private frame) | + * | .... | + * +-----------------------------------+ <= SP + * + * The FrameWithInstances is a struct with four fields: the saved FP, the return + * address, and the two instance slots; the shadow stack area is there only on + * Win64 and is unused by wasm but is part of the native ABI, with which the + * wasm ABI is mostly compatible. The slots for caller and callee instance are + * only populated by the instance switching code in inter-instance calls so that + * stack unwinding can keep track of the correct instance value for each frame, + * the instance not being obtainable from anywhere else. Nothing in the frame + * itself indicates directly whether the instance slots are valid - for that, + * the return address must be used to look up a CallSite structure that carries + * that information. + * + * The stack area above the return address is owned by the caller, which may + * deallocate the area on return or choose to reuse it for subsequent calls. + * (The baseline compiler allocates and frees the stack args area and the + * multi-value result area per call. Ion reuses the areas and allocates them as + * part of the overall activation frame when the procedure is entered; indeed, + * the multi-value return area can be anywhere within the caller's private + * frame, not necessarily directly above the stack args.) + * + * If the stack args area contain references, it is up to the callee's stack map + * to name the locations where those references exist, and the caller's stack + * map must not (redundantly) name those locations. (The callee's ownership of + * this area will be crucial for making tail calls work, as the types of the + * locations can change if the callee makes a tail call.) If pointer values are + * spilled by anyone into the Shadowstack area they will not be traced. + * + * References in the multi-return area are covered by the caller's map, as these + * slots outlive the call. + * + * The address "Before call", ie the part of the FrameWithInstances above the + * Frame, must be aligned to WasmStackAlignment, and everything follows from + * that, with padding inserted for alignment as required for stack arguments. In + * turn WasmStackAlignment is at least as large as the largest parameter type. + * + * The address of the multiple-results area is currently 8-byte aligned by Ion + * and its alignment in baseline is uncertain, see bug 1747787. Result values + * are stored packed within the area in fields whose size is given by + * ResultStackSize(ValType), this breaks alignment too. This all seems + * underdeveloped. + * + * In the wasm-internal ABI, the ARM64 PseudoStackPointer (PSP) is garbage on + * entry but must be synced with the real SP at the point the function returns. + * + * + * The Wasm Builtin ABIs. + * + * Also see `[SMDOC] Process-wide builtin thunk set` in WasmBuiltins.cpp. + * + * The *Wasm-builtin ABIs* comprise the ABIs used when wasm makes calls directly + * to the C++ runtime (but not to the JS interpreter), including instance + * methods, helpers for operations such as 64-bit division on 32-bit systems, + * allocation and writer barriers, conversions to/from JS values, special + * fast-path JS imports, and trap handling. + * + * The callee of a builtin call will always assume the C/C++ ABI. Therefore + * every volatile (caller-saves) register that wasm uses must be saved across + * the call, the stack must be aligned as for a C/C++-ABI call before the call, + * and any ABI registers the callee expect to have specific values must be set + * up (eg the frame pointer, if the C/C++ ABI assumes it is set). + * + * Most builtin calls are straightforward: the wasm caller knows that it is + * performing a call, and so it saves live registers, moves arguments into ABI + * locations, etc, before calling. Abstractions in the masm make sure to pass + * the instance pointer to an instance "method" call and to restore the + * InstanceReg and HeapReg after the call. In these straightforward cases, + * calling the builtin additionally amounts to: + * + * - exiting the wasm activation + * - adjusting parameter values to account for platform weirdness (FP arguments + * are handled differently in the C/C++ ABIs on ARM and x86-32 than in the + * Wasm ABI) + * - copying stack arguments into place for the C/C++ ABIs + * - making the call + * - adjusting the return values on return + * - re-entering the wasm activation and returning to the wasm caller + * + * The steps above are performed by the *builtin thunk* for the builtin and the + * builtin itself is said to be *thunked*. Going via the thunk is simple and, + * except for always having to copy stack arguments on x86-32 and the extra call + * in the thunk, close to as fast as we can make it without heroics. Except for + * the arithmetic helpers on 32-bit systems, most builtins are rarely used, are + * asm.js-specific, or are expensive anyway, and the overhead of the extra call + * doesn't matter. + * + * A few builtins for special purposes are *unthunked* and fall into two + * classes: they would normally be thunked but are used in circumstances where + * the VM is in an unusual state; or they do their work within the activation. + * + * In the former class, we find the debug trap handler, which must preserve all + * live registers because it is called in contexts where live registers have not + * been saved; argument coercion functions, which are called while a call frame + * is being built for a JS->Wasm or Wasm->JS call; and other routines that have + * special needs for constructing the call. These all exit the activation, but + * handle the exit specially. + * + * In the latter class, we find two functions that abandon the VM state and + * unwind the activation, HandleThrow and HandleTrap; and some debug print + * functions that do not affect the VM state at all. + * + * To summarize, when wasm calls a builtin thunk the stack will end up looking + * like this from within the C++ code: + * + * | | + * +-------------------------+ + * | Wasm frame | + * +-------------------------+ + * | Thunk frame (exit) | + * +-------------------------+ + * | Builtin frame (C++) | + * +-------------------------+ <= SP + * + * There is an assumption in the profiler (in initFromExitFP) that an exit has + * left precisely one frame on the stack for the thunk itself. There may be + * additional assumptions elsewhere, not yet found. + * + * Very occasionally, Wasm will call C++ without going through the builtin + * thunks, and this can be a source of problems. The one case I know about + * right now is that the JS pre-barrier filtering code is called directly from + * Wasm, see bug 1464157. + * + * + * Wasm stub ABIs. + * + * Also see `[SMDOC] Exported wasm functions and the jit-entry stubs` in + * WasmJS.cpp. + * + * The "stub ABIs" are not properly speaking ABIs themselves, but ABI + * converters. An "entry" stub calls in to wasm and an "exit" stub calls out + * from wasm. The entry stubs must convert from whatever data formats the + * caller has to wasm formats (and in the future must provide some kind of type + * checking for pointer types); the exit stubs convert from wasm formats to the + * callee's expected format. + * + * There are different entry paths from the JS interpreter (using the C++ ABI + * and data formats) and from jitted JS code (using the JIT ABI and data + * formats); indeed there is a "normal" JitEntry path ("JitEntry") that will + * perform argument and return value conversion, and the "fast" JitEntry path + * ("DirectCallFromJit") that is only used when it is known that the JIT will + * only pass and receive wasm-compatible data and no conversion is needed. + * + * Similarly, there are different exit paths to the interpreter (using the C++ + * ABI and data formats) and to JS JIT code (using the JIT ABI and data + * formats). Also, builtin calls described above are themselves a type of exit, + * and builtin thunks are properly a type of exit stub. + * + * Data conversions are difficult because the VM is in an intermediate state + * when they happen, we want them to be fast when possible, and some conversions + * can re-enter both JS code and wasm code. + */ + +#ifndef wasm_frame_h +#define wasm_frame_h + +#include "mozilla/Assertions.h" + +#include +#include +#include + +#include "jit/Registers.h" // For js::jit::ShadowStackSpace + +namespace js { +namespace wasm { + +class Instance; + +// Bit tag set when exiting wasm code in JitActivation's exitFP. +constexpr uintptr_t ExitFPTag = 0x1; + +// wasm::Frame represents the bytes pushed by the call instruction and the +// fixed prologue generated by wasm::GenerateCallablePrologue. +// +// Across all architectures it is assumed that, before the call instruction, the +// stack pointer is WasmStackAlignment-aligned. Thus after the prologue, and +// before the function has made its stack reservation, the stack alignment is +// sizeof(Frame) % WasmStackAlignment. +// +// During MacroAssembler code generation, the bytes pushed after the wasm::Frame +// are counted by masm.framePushed. Thus, the stack alignment at any point in +// time is (sizeof(wasm::Frame) + masm.framePushed) % WasmStackAlignment. + +class Frame { + // See GenerateCallableEpilogue for why this must be + // the first field of wasm::Frame (in a downward-growing stack). + // It's either the caller's Frame*, for wasm callers, or the JIT caller frame + // plus a tag otherwise. + uint8_t* callerFP_; + + // The return address pushed by the call (in the case of ARM/MIPS the return + // address is pushed by the first instruction of the prologue). + void* returnAddress_; + + public: + static constexpr uint32_t callerFPOffset() { + return offsetof(Frame, callerFP_); + } + static constexpr uint32_t returnAddressOffset() { + return offsetof(Frame, returnAddress_); + } + + uint8_t* returnAddress() const { + return reinterpret_cast(returnAddress_); + } + + void** addressOfReturnAddress() { + return reinterpret_cast(&returnAddress_); + } + + uint8_t* rawCaller() const { return callerFP_; } + + Frame* wasmCaller() const { return reinterpret_cast(callerFP_); } + + uint8_t* jitEntryCaller() const { return callerFP_; } + + static const Frame* fromUntaggedWasmExitFP(const void* savedFP) { + MOZ_ASSERT(!isExitFP(savedFP)); + return reinterpret_cast(savedFP); + } + + static bool isExitFP(const void* fp) { + return reinterpret_cast(fp) & ExitFPTag; + } + + static uint8_t* untagExitFP(const void* fp) { + MOZ_ASSERT(isExitFP(fp)); + return reinterpret_cast(reinterpret_cast(fp) & + ~ExitFPTag); + } + + static uint8_t* addExitFPTag(const Frame* fp) { + MOZ_ASSERT(!isExitFP(fp)); + return reinterpret_cast(reinterpret_cast(fp) | + ExitFPTag); + } +}; + +static_assert(!std::is_polymorphic_v, "Frame doesn't need a vtable."); +static_assert(sizeof(Frame) == 2 * sizeof(void*), + "Frame is a two pointer structure"); + +// Note that sizeof(FrameWithInstances) does not account for ShadowStackSpace. +// Use FrameWithInstances::sizeOf() if you are not incorporating +// ShadowStackSpace through other means (eg the ABIArgIter). + +class FrameWithInstances : public Frame { + // `ShadowStackSpace` bytes will be allocated here on Win64, at higher + // addresses than Frame and at lower addresses than the instance fields. + + // The instance area MUST be two pointers exactly. + Instance* calleeInstance_; + Instance* callerInstance_; + + public: + Instance* calleeInstance() { return calleeInstance_; } + Instance* callerInstance() { return callerInstance_; } + + constexpr static uint32_t sizeOf() { + return sizeof(wasm::FrameWithInstances) + js::jit::ShadowStackSpace; + } + + constexpr static uint32_t sizeOfInstanceFields() { + return sizeof(wasm::FrameWithInstances) - sizeof(wasm::Frame); + } + + constexpr static uint32_t calleeInstanceOffset() { + return offsetof(FrameWithInstances, calleeInstance_) + + js::jit::ShadowStackSpace; + } + + constexpr static uint32_t calleeInstanceOffsetWithoutFrame() { + return calleeInstanceOffset() - sizeof(wasm::Frame); + } + + constexpr static uint32_t callerInstanceOffset() { + return offsetof(FrameWithInstances, callerInstance_) + + js::jit::ShadowStackSpace; + } + + constexpr static uint32_t callerInstanceOffsetWithoutFrame() { + return callerInstanceOffset() - sizeof(wasm::Frame); + } +}; + +static_assert(FrameWithInstances::calleeInstanceOffsetWithoutFrame() == + js::jit::ShadowStackSpace, + "Callee instance stored right above the return address."); +static_assert(FrameWithInstances::callerInstanceOffsetWithoutFrame() == + js::jit::ShadowStackSpace + sizeof(void*), + "Caller instance stored right above the callee instance."); + +static_assert(FrameWithInstances::sizeOfInstanceFields() == 2 * sizeof(void*), + "There are only two additional slots"); + +#if defined(JS_CODEGEN_ARM64) +static_assert(sizeof(Frame) % 16 == 0, "frame is aligned"); +#endif + +} // namespace wasm +} // namespace js + +#endif // wasm_frame_h diff --git a/js/src/wasm/WasmFrameIter.cpp b/js/src/wasm/WasmFrameIter.cpp new file mode 100644 index 0000000000..8b57fd42fa --- /dev/null +++ b/js/src/wasm/WasmFrameIter.cpp @@ -0,0 +1,1764 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2014 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmFrameIter.h" + +#include "jit/JitFrames.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/JSContext.h" +#include "wasm/WasmDebugFrame.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmInstanceData.h" +#include "wasm/WasmIntrinsicGenerated.h" +#include "wasm/WasmStubs.h" + +#include "jit/MacroAssembler-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::DebugOnly; +using mozilla::Maybe; + +static Instance* ExtractCallerInstanceFromFrameWithInstances(Frame* fp) { + return *reinterpret_cast( + reinterpret_cast(fp) + + FrameWithInstances::callerInstanceOffset()); +} + +static const Instance* ExtractCalleeInstanceFromFrameWithInstances( + const Frame* fp) { + return *reinterpret_cast( + reinterpret_cast(fp) + + FrameWithInstances::calleeInstanceOffset()); +} + +/*****************************************************************************/ +// WasmFrameIter implementation + +WasmFrameIter::WasmFrameIter(JitActivation* activation, wasm::Frame* fp) + : activation_(activation), + code_(nullptr), + codeRange_(nullptr), + lineOrBytecode_(0), + fp_(fp ? fp : activation->wasmExitFP()), + instance_(nullptr), + unwoundCallerFP_(nullptr), + unwoundJitFrameType_(), + unwind_(Unwind::False), + unwoundAddressOfReturnAddress_(nullptr), + resumePCinCurrentFrame_(nullptr) { + MOZ_ASSERT(fp_); + instance_ = GetNearestEffectiveInstance(fp_); + + // When the stack is captured during a trap (viz., to create the .stack + // for an Error object), use the pc/bytecode information captured by the + // signal handler in the runtime. Take care not to use this trap unwind + // state for wasm frames in the middle of a JitActivation, i.e., wasm frames + // that called into JIT frames before the trap. + + if (activation->isWasmTrapping() && fp_ == activation->wasmExitFP()) { + const TrapData& trapData = activation->wasmTrapData(); + void* unwoundPC = trapData.unwoundPC; + + code_ = &instance_->code(); + MOZ_ASSERT(code_ == LookupCode(unwoundPC)); + + codeRange_ = code_->lookupFuncRange(unwoundPC); + MOZ_ASSERT(codeRange_); + + lineOrBytecode_ = trapData.bytecodeOffset; + + MOZ_ASSERT(!done()); + return; + } + + // Otherwise, execution exits wasm code via an exit stub which sets exitFP + // to the exit stub's frame. Thus, in this case, we want to start iteration + // at the caller of the exit frame, whose Code, CodeRange and CallSite are + // indicated by the returnAddress of the exit stub's frame. If the caller + // was Ion, we can just skip the wasm frames. + + popFrame(); + MOZ_ASSERT(!done() || unwoundCallerFP_); +} + +bool WasmFrameIter::done() const { + MOZ_ASSERT(!!fp_ == !!code_); + MOZ_ASSERT(!!fp_ == !!codeRange_); + return !fp_; +} + +void WasmFrameIter::operator++() { + MOZ_ASSERT(!done()); + + // When the iterator is set to unwind, each time the iterator pops a frame, + // the JitActivation is updated so that the just-popped frame is no longer + // visible. This is necessary since Debugger::onLeaveFrame is called before + // popping each frame and, once onLeaveFrame is called for a given frame, + // that frame must not be visible to subsequent stack iteration (or it + // could be added as a "new" frame just as it becomes garbage). When the + // frame is trapping, then exitFP is included in the callstack (otherwise, + // it is skipped, as explained above). So to unwind the innermost frame, we + // just clear the trapping state. + + if (unwind_ == Unwind::True) { + if (activation_->isWasmTrapping()) { + activation_->finishWasmTrap(); + } + activation_->setWasmExitFP(fp_); + } + + popFrame(); +} + +static inline void AssertDirectJitCall(const void* fp) { + // Called via an inlined fast JIT to wasm call: in this case, FP is + // pointing in the middle of the exit frame, right before the exit + // footer; ensure the exit frame type is the expected one. +#ifdef DEBUG + auto* jitCaller = (ExitFrameLayout*)fp; + MOZ_ASSERT(jitCaller->footer()->type() == + jit::ExitFrameType::DirectWasmJitCall); +#endif +} + +void WasmFrameIter::popFrame() { + uint8_t* returnAddress = fp_->returnAddress(); + code_ = LookupCode(returnAddress, &codeRange_); + + if (!code_) { + // This is a direct call from the jit into the wasm function's body. The + // call stack resembles this at this point: + // + // |---------------------| + // | JIT FRAME | + // | JIT FAKE EXIT FRAME | <-- fp_->callerFP_ + // | WASM FRAME | <-- fp_ + // |---------------------| + // + // fp_->callerFP_ points to the fake exit frame set up by the jit caller, + // and the return-address-to-fp is in JIT code, thus doesn't belong to any + // wasm instance's code (in particular, there's no associated CodeRange). + // Mark the frame as such. + AssertDirectJitCall(fp_->jitEntryCaller()); + + unwoundCallerFP_ = fp_->jitEntryCaller(); + unwoundJitFrameType_.emplace(FrameType::Exit); + + if (unwind_ == Unwind::True) { + activation_->setJSExitFP(unwoundCallerFP()); + unwoundAddressOfReturnAddress_ = fp_->addressOfReturnAddress(); + } + + fp_ = nullptr; + code_ = nullptr; + codeRange_ = nullptr; + + MOZ_ASSERT(done()); + return; + } + + MOZ_ASSERT(codeRange_); + + Frame* prevFP = fp_; + fp_ = fp_->wasmCaller(); + resumePCinCurrentFrame_ = returnAddress; + + if (codeRange_->isInterpEntry()) { + // Interpreter entry has a simple frame, record FP from it. + unwoundCallerFP_ = reinterpret_cast(fp_); + + fp_ = nullptr; + code_ = nullptr; + codeRange_ = nullptr; + + if (unwind_ == Unwind::True) { + // We're exiting via the interpreter entry; we can safely reset + // exitFP. + activation_->setWasmExitFP(nullptr); + unwoundAddressOfReturnAddress_ = prevFP->addressOfReturnAddress(); + } + + MOZ_ASSERT(done()); + return; + } + + if (codeRange_->isJitEntry()) { + // This wasm function has been called through the generic JIT entry by + // a JIT caller, so the call stack resembles this: + // + // |---------------------| + // | JIT FRAME | + // | JSJIT TO WASM EXIT | <-- fp_ + // | WASM JIT ENTRY | <-- prevFP (already unwound) + // | WASM FRAME | (already unwound) + // |---------------------| + // + // The next value of FP is just a regular jit frame used as a marker to + // know that we should transition to a JSJit frame iterator. + unwoundCallerFP_ = reinterpret_cast(fp_); + unwoundJitFrameType_.emplace(FrameType::JSJitToWasm); + + fp_ = nullptr; + code_ = nullptr; + codeRange_ = nullptr; + + if (unwind_ == Unwind::True) { + activation_->setJSExitFP(unwoundCallerFP()); + unwoundAddressOfReturnAddress_ = prevFP->addressOfReturnAddress(); + } + + MOZ_ASSERT(done()); + return; + } + + MOZ_ASSERT(codeRange_->kind() == CodeRange::Function); + + const CallSite* callsite = code_->lookupCallSite(returnAddress); + MOZ_ASSERT(callsite); + + if (callsite->mightBeCrossInstance()) { + instance_ = ExtractCallerInstanceFromFrameWithInstances(prevFP); + } + + MOZ_ASSERT(code_ == &instance()->code()); + lineOrBytecode_ = callsite->lineOrBytecode(); + + MOZ_ASSERT(!done()); +} + +const char* WasmFrameIter::filename() const { + MOZ_ASSERT(!done()); + return code_->metadata().filename.get(); +} + +const char16_t* WasmFrameIter::displayURL() const { + MOZ_ASSERT(!done()); + return code_->metadata().displayURL(); +} + +bool WasmFrameIter::mutedErrors() const { + MOZ_ASSERT(!done()); + return code_->metadata().mutedErrors(); +} + +JSAtom* WasmFrameIter::functionDisplayAtom() const { + MOZ_ASSERT(!done()); + + JSContext* cx = activation_->cx(); + JSAtom* atom = instance()->getFuncDisplayAtom(cx, codeRange_->funcIndex()); + if (!atom) { + cx->clearPendingException(); + return cx->names().empty; + } + + return atom; +} + +unsigned WasmFrameIter::lineOrBytecode() const { + MOZ_ASSERT(!done()); + return lineOrBytecode_; +} + +uint32_t WasmFrameIter::funcIndex() const { + MOZ_ASSERT(!done()); + return codeRange_->funcIndex(); +} + +unsigned WasmFrameIter::computeLine(uint32_t* column) const { + if (instance()->isAsmJS()) { + if (column) { + *column = 1; + } + return lineOrBytecode_; + } + + // As a terrible hack to avoid changing the tons of places that pass around + // (url, line, column) tuples to instead passing around a Variant that + // stores a (url, func-index, bytecode-offset) tuple for wasm frames, + // wasm stuffs its tuple into the existing (url, line, column) tuple, + // tagging the high bit of the column to indicate "this is a wasm frame". + // When knowing clients see this bit, they shall render the tuple + // (url, line, column|bit) as "url:wasm-function[column]:0xline" according + // to the WebAssembly Web API's Developer-Facing Display Conventions. + // https://webassembly.github.io/spec/web-api/index.html#conventions + // The wasm bytecode offset continues to be passed as the JS line to avoid + // breaking existing devtools code written when this used to be the case. + + MOZ_ASSERT(!(codeRange_->funcIndex() & ColumnBit)); + if (column) { + *column = codeRange_->funcIndex() | ColumnBit; + } + return lineOrBytecode_; +} + +void** WasmFrameIter::unwoundAddressOfReturnAddress() const { + MOZ_ASSERT(done()); + MOZ_ASSERT(unwind_ == Unwind::True); + MOZ_ASSERT(unwoundAddressOfReturnAddress_); + return unwoundAddressOfReturnAddress_; +} + +bool WasmFrameIter::debugEnabled() const { + MOZ_ASSERT(!done()); + + // Only non-imported functions can have debug frames. + // + // Metadata::debugEnabled is only set if debugging is actually enabled (both + // requested, and available via baseline compilation), and Tier::Debug code + // will be available. + return code_->metadata().debugEnabled && + codeRange_->funcIndex() >= + code_->metadata(Tier::Debug).funcImports.length(); +} + +DebugFrame* WasmFrameIter::debugFrame() const { + MOZ_ASSERT(!done()); + return DebugFrame::from(fp_); +} + +bool WasmFrameIter::hasUnwoundJitFrame() const { + return unwoundCallerFP_ && unwoundJitFrameType_.isSome(); +} + +jit::FrameType WasmFrameIter::unwoundJitFrameType() const { + MOZ_ASSERT(unwoundCallerFP_); + MOZ_ASSERT(unwoundJitFrameType_.isSome()); + return *unwoundJitFrameType_; +} + +uint8_t* WasmFrameIter::resumePCinCurrentFrame() const { + if (resumePCinCurrentFrame_) { + return resumePCinCurrentFrame_; + } + MOZ_ASSERT(activation_->isWasmTrapping()); + // The next instruction is the instruction following the trap instruction. + return (uint8_t*)activation_->wasmTrapData().resumePC; +} + +/*****************************************************************************/ +// Prologue/epilogue code generation + +// These constants reflect statically-determined offsets in the +// prologue/epilogue. The offsets are dynamically asserted during code +// generation. +#if defined(JS_CODEGEN_X64) +static const unsigned PushedRetAddr = 0; +static const unsigned PushedFP = 1; +static const unsigned SetFP = 4; +static const unsigned PoppedFP = 0; +static const unsigned PoppedFPJitEntry = 0; +#elif defined(JS_CODEGEN_X86) +static const unsigned PushedRetAddr = 0; +static const unsigned PushedFP = 1; +static const unsigned SetFP = 3; +static const unsigned PoppedFP = 0; +static const unsigned PoppedFPJitEntry = 0; +#elif defined(JS_CODEGEN_ARM) +static const unsigned BeforePushRetAddr = 0; +static const unsigned PushedRetAddr = 4; +static const unsigned PushedFP = 8; +static const unsigned SetFP = 12; +static const unsigned PoppedFP = 0; +static const unsigned PoppedFPJitEntry = 0; +#elif defined(JS_CODEGEN_ARM64) +// On ARM64 we do not use push or pop; the prologues and epilogues are +// structured differently due to restrictions on SP alignment. Even so, +// PushedRetAddr and PushedFP are used in some restricted contexts +// and must be superficially meaningful. +static const unsigned BeforePushRetAddr = 0; +static const unsigned PushedRetAddr = 8; +static const unsigned PushedFP = 12; +static const unsigned SetFP = 16; +static const unsigned PoppedFP = 4; +static const unsigned PoppedFPJitEntry = 8; +static_assert(BeforePushRetAddr == 0, "Required by StartUnwinding"); +static_assert(PushedFP > PushedRetAddr, "Required by StartUnwinding"); +#elif defined(JS_CODEGEN_MIPS64) +static const unsigned PushedRetAddr = 8; +static const unsigned PushedFP = 16; +static const unsigned SetFP = 20; +static const unsigned PoppedFP = 4; +static const unsigned PoppedFPJitEntry = 0; +#elif defined(JS_CODEGEN_LOONG64) +static const unsigned PushedRetAddr = 8; +static const unsigned PushedFP = 16; +static const unsigned SetFP = 20; +static const unsigned PoppedFP = 4; +static const unsigned PoppedFPJitEntry = 0; +#elif defined(JS_CODEGEN_RISCV64) +static const unsigned PushedRetAddr = 8; +static const unsigned PushedFP = 16; +static const unsigned SetFP = 20; +static const unsigned PoppedFP = 4; +static const unsigned PoppedFPJitEntry = 0; +#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32) +// Synthetic values to satisfy asserts and avoid compiler warnings. +static const unsigned PushedRetAddr = 0; +static const unsigned PushedFP = 1; +static const unsigned SetFP = 2; +static const unsigned PoppedFP = 3; +static const unsigned PoppedFPJitEntry = 4; +#else +# error "Unknown architecture!" +#endif + +static void LoadActivation(MacroAssembler& masm, const Register& dest) { + // WasmCall pushes a JitActivation. + masm.loadPtr(Address(InstanceReg, wasm::Instance::offsetOfCx()), dest); + masm.loadPtr(Address(dest, JSContext::offsetOfActivation()), dest); +} + +void wasm::SetExitFP(MacroAssembler& masm, ExitReason reason, + Register scratch) { + MOZ_ASSERT(!reason.isNone()); + + LoadActivation(masm, scratch); + + masm.store32( + Imm32(reason.encode()), + Address(scratch, JitActivation::offsetOfEncodedWasmExitReason())); + + masm.orPtr(Imm32(ExitFPTag), FramePointer); + masm.storePtr(FramePointer, + Address(scratch, JitActivation::offsetOfPackedExitFP())); + masm.andPtr(Imm32(int32_t(~ExitFPTag)), FramePointer); +} + +void wasm::ClearExitFP(MacroAssembler& masm, Register scratch) { + LoadActivation(masm, scratch); + masm.storePtr(ImmWord(0x0), + Address(scratch, JitActivation::offsetOfPackedExitFP())); + masm.store32( + Imm32(0x0), + Address(scratch, JitActivation::offsetOfEncodedWasmExitReason())); +} + +static void GenerateCallablePrologue(MacroAssembler& masm, uint32_t* entry) { + AutoCreatedBy acb(masm, "GenerateCallablePrologue"); + masm.setFramePushed(0); + + // ProfilingFrameIterator needs to know the offsets of several key + // instructions from entry. To save space, we make these offsets static + // constants and assert that they match the actual codegen below. On ARM, + // this requires AutoForbidPoolsAndNops to prevent a constant pool from being + // randomly inserted between two instructions. + +#if defined(JS_CODEGEN_MIPS64) + { + *entry = masm.currentOffset(); + + masm.ma_push(ra); + MOZ_ASSERT_IF(!masm.oom(), PushedRetAddr == masm.currentOffset() - *entry); + masm.ma_push(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), PushedFP == masm.currentOffset() - *entry); + masm.moveStackPtrTo(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); + } +#elif defined(JS_CODEGEN_LOONG64) + { + *entry = masm.currentOffset(); + + masm.ma_push(ra); + MOZ_ASSERT_IF(!masm.oom(), PushedRetAddr == masm.currentOffset() - *entry); + masm.ma_push(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), PushedFP == masm.currentOffset() - *entry); + masm.moveStackPtrTo(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); + } +#elif defined(JS_CODEGEN_RISCV64) + { + *entry = masm.currentOffset(); + BlockTrampolinePoolScope block_trampoline_pool(&masm, 5); + masm.ma_push(ra); + MOZ_ASSERT_IF(!masm.oom(), PushedRetAddr == masm.currentOffset() - *entry); + masm.ma_push(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), PushedFP == masm.currentOffset() - *entry); + masm.moveStackPtrTo(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); + } +#elif defined(JS_CODEGEN_ARM64) + { + // We do not use the PseudoStackPointer. However, we may be called in a + // context -- compilation using Ion -- in which the PseudoStackPointer is + // in use. Rather than risk confusion in the uses of `masm` here, let's + // just switch in the real SP, do what we need to do, and restore the + // existing setting afterwards. + const vixl::Register stashedSPreg = masm.GetStackPointer64(); + masm.SetStackPointer64(vixl::sp); + + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 4); + + *entry = masm.currentOffset(); + + masm.Sub(sp, sp, sizeof(Frame)); + masm.Str(ARMRegister(lr, 64), MemOperand(sp, Frame::returnAddressOffset())); + MOZ_ASSERT_IF(!masm.oom(), PushedRetAddr == masm.currentOffset() - *entry); + masm.Str(ARMRegister(FramePointer, 64), + MemOperand(sp, Frame::callerFPOffset())); + MOZ_ASSERT_IF(!masm.oom(), PushedFP == masm.currentOffset() - *entry); + masm.Mov(ARMRegister(FramePointer, 64), sp); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); + + // And restore the SP-reg setting, per comment above. + masm.SetStackPointer64(stashedSPreg); + } +#else + { +# if defined(JS_CODEGEN_ARM) + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 3); + + *entry = masm.currentOffset(); + + static_assert(BeforePushRetAddr == 0); + masm.push(lr); +# else + *entry = masm.currentOffset(); + // The x86/x64 call instruction pushes the return address. +# endif + + MOZ_ASSERT_IF(!masm.oom(), PushedRetAddr == masm.currentOffset() - *entry); + masm.push(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), PushedFP == masm.currentOffset() - *entry); + masm.moveStackPtrTo(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - *entry); + } +#endif +} + +static void GenerateCallableEpilogue(MacroAssembler& masm, unsigned framePushed, + ExitReason reason, uint32_t* ret) { + AutoCreatedBy acb(masm, "GenerateCallableEpilogue"); + + if (framePushed) { + masm.freeStack(framePushed); + } + + if (!reason.isNone()) { + ClearExitFP(masm, ABINonArgReturnVolatileReg); + } + + DebugOnly poppedFP{}; + +#if defined(JS_CODEGEN_MIPS64) + + masm.loadPtr(Address(StackPointer, Frame::callerFPOffset()), FramePointer); + poppedFP = masm.currentOffset(); + masm.loadPtr(Address(StackPointer, Frame::returnAddressOffset()), ra); + + *ret = masm.currentOffset(); + masm.as_jr(ra); + masm.addToStackPtr(Imm32(sizeof(Frame))); + +#elif defined(JS_CODEGEN_LOONG64) + + masm.loadPtr(Address(StackPointer, Frame::callerFPOffset()), FramePointer); + poppedFP = masm.currentOffset(); + masm.loadPtr(Address(StackPointer, Frame::returnAddressOffset()), ra); + + *ret = masm.currentOffset(); + masm.addToStackPtr(Imm32(sizeof(Frame))); + masm.as_jirl(zero, ra, BOffImm16(0)); + +#elif defined(JS_CODEGEN_RISCV64) + { + BlockTrampolinePoolScope block_trampoline_pool(&masm, 20); + masm.loadPtr(Address(StackPointer, Frame::callerFPOffset()), FramePointer); + poppedFP = masm.currentOffset(); + masm.loadPtr(Address(StackPointer, Frame::returnAddressOffset()), ra); + + *ret = masm.currentOffset(); + masm.addToStackPtr(Imm32(sizeof(Frame))); + masm.jalr(zero, ra, 0); + masm.nop(); + } +#elif defined(JS_CODEGEN_ARM64) + + // See comment at equivalent place in |GenerateCallablePrologue| above. + const vixl::Register stashedSPreg = masm.GetStackPointer64(); + masm.SetStackPointer64(vixl::sp); + + AutoForbidPoolsAndNops afp(&masm, /* number of instructions in scope = */ 5); + + masm.Ldr(ARMRegister(FramePointer, 64), + MemOperand(sp, Frame::callerFPOffset())); + poppedFP = masm.currentOffset(); + + masm.Ldr(ARMRegister(lr, 64), MemOperand(sp, Frame::returnAddressOffset())); + *ret = masm.currentOffset(); + + masm.Add(sp, sp, sizeof(Frame)); + + // Reinitialise PSP from SP. This is less than elegant because the prologue + // operates on the raw stack pointer SP and does not keep the PSP in sync. + // We can't use initPseudoStackPtr here because we just set up masm to not + // use it. Hence we have to do it "by hand". + masm.Mov(PseudoStackPointer64, vixl::sp); + + masm.Ret(ARMRegister(lr, 64)); + + // See comment at equivalent place in |GenerateCallablePrologue| above. + masm.SetStackPointer64(stashedSPreg); + +#else + // Forbid pools for the same reason as described in GenerateCallablePrologue. +# if defined(JS_CODEGEN_ARM) + AutoForbidPoolsAndNops afp(&masm, /* number of instructions in scope = */ 6); +# endif + + // There is an important ordering constraint here: fp must be repointed to + // the caller's frame before any field of the frame currently pointed to by + // fp is popped: asynchronous signal handlers (which use stack space + // starting at sp) could otherwise clobber these fields while they are still + // accessible via fp (fp fields are read during frame iteration which is + // *also* done asynchronously). + + masm.pop(FramePointer); + poppedFP = masm.currentOffset(); + + *ret = masm.currentOffset(); + masm.ret(); + +#endif + + MOZ_ASSERT_IF(!masm.oom(), PoppedFP == *ret - poppedFP); +} + +void wasm::GenerateFunctionPrologue(MacroAssembler& masm, + const CallIndirectId& callIndirectId, + const Maybe& tier1FuncIndex, + FuncOffsets* offsets) { + AutoCreatedBy acb(masm, "wasm::GenerateFunctionPrologue"); + + // We are going to generate this code layout: + // --------------------------------------------- + // checked call entry: callable prologue + // check signature + // jump functionBody ──┐ + // unchecked call entry: callable prologue │ + // functionBody <─────┘ + // ----------------------------------------------- + // checked call entry - used for call_indirect when we have to check the + // signature. + // + // unchecked call entry - used for regular direct same-instance calls. + + // The checked call entry is a call target, so must have CodeAlignment. + // Its offset is normally zero. + static_assert(WasmCheckedCallEntryOffset % CodeAlignment == 0, + "code aligned"); + + // Flush pending pools so they do not get dumped between the 'begin' and + // 'uncheckedCallEntry' offsets since the difference must be less than + // UINT8_MAX to be stored in CodeRange::funcbeginToUncheckedCallEntry_. + // (Pending pools can be large.) + masm.flushBuffer(); + masm.haltingAlign(CodeAlignment); + + Label functionBody; + + offsets->begin = masm.currentOffset(); + + // Only first-class functions (those that can be referenced in a table) need + // the checked call prologue w/ signature check. It is impossible to perform + // a checked call otherwise. + // + // asm.js function tables are homogeneous and don't need a signature check. + // However, they can be put in tables which expect a checked call entry point, + // so we generate a no-op entry point for consistency. If asm.js performance + // was important we could refine this in the future. + if (callIndirectId.kind() != CallIndirectIdKind::None) { + // Generate checked call entry. The BytecodeOffset of the trap is fixed up + // to be the bytecode offset of the callsite by + // JitActivation::startWasmTrap. + MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() - offsets->begin == + WasmCheckedCallEntryOffset); + uint32_t dummy; + GenerateCallablePrologue(masm, &dummy); + + switch (callIndirectId.kind()) { + case CallIndirectIdKind::Global: { + Register scratch = WasmTableCallScratchReg0; + masm.loadPtr( + Address(InstanceReg, Instance::offsetInData( + callIndirectId.instanceDataOffset())), + scratch); + masm.branchPtr(Assembler::Condition::Equal, WasmTableCallSigReg, + scratch, &functionBody); + masm.wasmTrap(Trap::IndirectCallBadSig, BytecodeOffset(0)); + break; + } + case CallIndirectIdKind::Immediate: { + masm.branch32(Assembler::Condition::Equal, WasmTableCallSigReg, + Imm32(callIndirectId.immediate()), &functionBody); + masm.wasmTrap(Trap::IndirectCallBadSig, BytecodeOffset(0)); + break; + } + case CallIndirectIdKind::AsmJS: + masm.jump(&functionBody); + break; + case CallIndirectIdKind::None: + break; + } + + // The preceding code may have generated a small constant pool to support + // the comparison in the signature check. But if we flush the pool here we + // will also force the creation of an unused branch veneer in the pool for + // the jump to functionBody from the signature check on some platforms, thus + // needlessly inflating the size of the prologue. + // + // On no supported platform that uses a pool (arm, arm64) is there any risk + // at present of that branch or other elements in the pool going out of + // range while we're generating the following padding and prologue, + // therefore no pool elements will be emitted in the prologue, therefore it + // is safe not to flush here. + // + // We assert that this holds at runtime by comparing the expected entry + // offset to the recorded ditto; if they are not the same then + // GenerateCallablePrologue flushed a pool before the prologue code, + // contrary to assumption. + + masm.nopAlign(CodeAlignment); + } + + // Generate unchecked call entry: + DebugOnly expectedEntry = masm.currentOffset(); + GenerateCallablePrologue(masm, &offsets->uncheckedCallEntry); + MOZ_ASSERT(expectedEntry == offsets->uncheckedCallEntry); + masm.bind(&functionBody); +#ifdef JS_CODEGEN_ARM64 + // GenerateCallablePrologue creates a prologue which operates on the raw + // stack pointer and does not keep the PSP in sync. So we have to resync it + // here. But we can't use initPseudoStackPtr here because masm may not be + // set up to use it, depending on which compiler is in use. Hence do it + // "manually". + masm.Mov(PseudoStackPointer64, vixl::sp); +#endif + + // See comment block in WasmCompile.cpp for an explanation tiering. + if (tier1FuncIndex) { + Register scratch = ABINonArgReg0; + masm.loadPtr(Address(InstanceReg, Instance::offsetOfJumpTable()), scratch); + masm.jump(Address(scratch, *tier1FuncIndex * sizeof(uintptr_t))); + } + + offsets->tierEntry = masm.currentOffset(); + + MOZ_ASSERT(masm.framePushed() == 0); +} + +void wasm::GenerateFunctionEpilogue(MacroAssembler& masm, unsigned framePushed, + FuncOffsets* offsets) { + // Inverse of GenerateFunctionPrologue: + MOZ_ASSERT(masm.framePushed() == framePushed); + GenerateCallableEpilogue(masm, framePushed, ExitReason::None(), + &offsets->ret); + MOZ_ASSERT(masm.framePushed() == 0); +} + +void wasm::GenerateExitPrologue(MacroAssembler& masm, unsigned framePushed, + ExitReason reason, CallableOffsets* offsets) { + masm.haltingAlign(CodeAlignment); + + GenerateCallablePrologue(masm, &offsets->begin); + + // This frame will be exiting compiled code to C++ so record the fp and + // reason in the JitActivation so the frame iterators can unwind. + SetExitFP(masm, reason, ABINonArgReturnVolatileReg); + + MOZ_ASSERT(masm.framePushed() == 0); + masm.reserveStack(framePushed); +} + +void wasm::GenerateExitEpilogue(MacroAssembler& masm, unsigned framePushed, + ExitReason reason, CallableOffsets* offsets) { + // Inverse of GenerateExitPrologue: + MOZ_ASSERT(masm.framePushed() == framePushed); + GenerateCallableEpilogue(masm, framePushed, reason, &offsets->ret); + MOZ_ASSERT(masm.framePushed() == 0); +} + +static void AssertNoWasmExitFPInJitExit(MacroAssembler& masm) { + // As a general stack invariant, if Activation::packedExitFP is tagged as + // wasm, it must point to a valid wasm::Frame. The JIT exit stub calls into + // JIT code and thus does not really exit, thus, when entering/leaving the + // JIT exit stub from/to normal wasm code, packedExitFP is not tagged wasm. +#ifdef DEBUG + Register scratch = ABINonArgReturnReg0; + LoadActivation(masm, scratch); + + Label ok; + masm.branchTestPtr(Assembler::Zero, + Address(scratch, JitActivation::offsetOfPackedExitFP()), + Imm32(ExitFPTag), &ok); + masm.breakpoint(); + masm.bind(&ok); +#endif +} + +void wasm::GenerateJitExitPrologue(MacroAssembler& masm, unsigned framePushed, + CallableOffsets* offsets) { + masm.haltingAlign(CodeAlignment); + + GenerateCallablePrologue(masm, &offsets->begin); + AssertNoWasmExitFPInJitExit(masm); + + MOZ_ASSERT(masm.framePushed() == 0); + masm.reserveStack(framePushed); +} + +void wasm::GenerateJitExitEpilogue(MacroAssembler& masm, unsigned framePushed, + CallableOffsets* offsets) { + // Inverse of GenerateJitExitPrologue: + MOZ_ASSERT(masm.framePushed() == framePushed); + AssertNoWasmExitFPInJitExit(masm); + GenerateCallableEpilogue(masm, framePushed, ExitReason::None(), + &offsets->ret); + MOZ_ASSERT(masm.framePushed() == 0); +} + +void wasm::GenerateJitEntryPrologue(MacroAssembler& masm, + CallableOffsets* offsets) { + masm.haltingAlign(CodeAlignment); + + { + // Push the return address. +#if defined(JS_CODEGEN_ARM) + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 3); + offsets->begin = masm.currentOffset(); + static_assert(BeforePushRetAddr == 0); + masm.push(lr); +#elif defined(JS_CODEGEN_MIPS64) + offsets->begin = masm.currentOffset(); + masm.push(ra); +#elif defined(JS_CODEGEN_LOONG64) + offsets->begin = masm.currentOffset(); + masm.push(ra); +#elif defined(JS_CODEGEN_RISCV64) + BlockTrampolinePoolScope block_trampoline_pool(&masm, 10); + offsets->begin = masm.currentOffset(); + masm.push(ra); +#elif defined(JS_CODEGEN_ARM64) + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 4); + offsets->begin = masm.currentOffset(); + static_assert(BeforePushRetAddr == 0); + // Subtract from SP first as SP must be aligned before offsetting. + masm.Sub(sp, sp, 16); + static_assert(JitFrameLayout::offsetOfReturnAddress() == 8); + masm.Str(ARMRegister(lr, 64), MemOperand(sp, 8)); +#else + // The x86/x64 call instruction pushes the return address. + offsets->begin = masm.currentOffset(); +#endif + MOZ_ASSERT_IF(!masm.oom(), + PushedRetAddr == masm.currentOffset() - offsets->begin); + // Save jit frame pointer, so unwinding from wasm to jit frames is trivial. +#if defined(JS_CODEGEN_ARM64) + static_assert(JitFrameLayout::offsetOfCallerFramePtr() == 0); + masm.Str(ARMRegister(FramePointer, 64), MemOperand(sp, 0)); +#else + masm.Push(FramePointer); +#endif + MOZ_ASSERT_IF(!masm.oom(), + PushedFP == masm.currentOffset() - offsets->begin); + + masm.moveStackPtrTo(FramePointer); + MOZ_ASSERT_IF(!masm.oom(), SetFP == masm.currentOffset() - offsets->begin); + } + + masm.setFramePushed(0); +} + +void wasm::GenerateJitEntryEpilogue(MacroAssembler& masm, + CallableOffsets* offsets) { + DebugOnly poppedFP{}; +#ifdef JS_CODEGEN_ARM64 + RegisterOrSP sp = masm.getStackPointer(); + AutoForbidPoolsAndNops afp(&masm, + /* number of instructions in scope = */ 5); + masm.loadPtr(Address(sp, 8), lr); + masm.loadPtr(Address(sp, 0), FramePointer); + poppedFP = masm.currentOffset(); + + masm.addToStackPtr(Imm32(2 * sizeof(void*))); + // Copy SP into PSP to enforce return-point invariants (SP == PSP). + // `addToStackPtr` won't sync them because SP is the active pointer here. + // For the same reason, we can't use initPseudoStackPtr to do the sync, so + // we have to do it "by hand". Omitting this causes many tests to segfault. + masm.moveStackPtrTo(PseudoStackPointer); + + offsets->ret = masm.currentOffset(); + masm.Ret(ARMRegister(lr, 64)); + masm.setFramePushed(0); +#else + // Forbid pools for the same reason as described in GenerateCallablePrologue. +# if defined(JS_CODEGEN_ARM) + AutoForbidPoolsAndNops afp(&masm, /* number of instructions in scope = */ 2); +# endif + + masm.pop(FramePointer); + poppedFP = masm.currentOffset(); + + offsets->ret = masm.currentOffset(); + masm.ret(); +#endif + MOZ_ASSERT_IF(!masm.oom(), PoppedFPJitEntry == offsets->ret - poppedFP); +} + +/*****************************************************************************/ +// ProfilingFrameIterator + +ProfilingFrameIterator::ProfilingFrameIterator() + : code_(nullptr), + codeRange_(nullptr), + callerFP_(nullptr), + callerPC_(nullptr), + stackAddress_(nullptr), + unwoundJitCallerFP_(nullptr), + exitReason_(ExitReason::Fixed::None) { + MOZ_ASSERT(done()); +} + +ProfilingFrameIterator::ProfilingFrameIterator(const JitActivation& activation) + : code_(nullptr), + codeRange_(nullptr), + callerFP_(nullptr), + callerPC_(nullptr), + stackAddress_(nullptr), + unwoundJitCallerFP_(nullptr), + exitReason_(activation.wasmExitReason()) { + initFromExitFP(activation.wasmExitFP()); +} + +ProfilingFrameIterator::ProfilingFrameIterator(const Frame* fp) + : code_(nullptr), + codeRange_(nullptr), + callerFP_(nullptr), + callerPC_(nullptr), + stackAddress_(nullptr), + unwoundJitCallerFP_(nullptr), + exitReason_(ExitReason::Fixed::ImportJit) { + MOZ_ASSERT(fp); + initFromExitFP(fp); +} + +static inline void AssertMatchesCallSite(void* callerPC, uint8_t* callerFP) { +#ifdef DEBUG + const CodeRange* callerCodeRange; + const Code* code = LookupCode(callerPC, &callerCodeRange); + + if (!code) { + AssertDirectJitCall(callerFP); + return; + } + + MOZ_ASSERT(callerCodeRange); + + if (callerCodeRange->isInterpEntry()) { + // callerFP is the value of the frame pointer register when we were called + // from C++. + return; + } + + if (callerCodeRange->isJitEntry()) { + MOZ_ASSERT(callerFP != nullptr); + return; + } + + const CallSite* callsite = code->lookupCallSite(callerPC); + MOZ_ASSERT(callsite); +#endif +} + +void ProfilingFrameIterator::initFromExitFP(const Frame* fp) { + MOZ_ASSERT(fp); + stackAddress_ = (void*)fp; + endStackAddress_ = stackAddress_; + code_ = LookupCode(fp->returnAddress(), &codeRange_); + + if (!code_) { + // This is a direct call from the JIT, the caller FP is pointing to the JIT + // caller's frame. + AssertDirectJitCall(fp->jitEntryCaller()); + + unwoundJitCallerFP_ = fp->jitEntryCaller(); + MOZ_ASSERT(done()); + return; + } + + MOZ_ASSERT(codeRange_); + + // Since we don't have the pc for fp, start unwinding at the caller of fp. + // This means that the innermost frame is skipped. This is fine because: + // - for import exit calls, the innermost frame is a thunk, so the first + // frame that shows up is the function calling the import; + // - for Math and other builtin calls, we note the absence of an exit + // reason and inject a fake "builtin" frame; and + switch (codeRange_->kind()) { + case CodeRange::InterpEntry: + callerPC_ = nullptr; + callerFP_ = nullptr; + break; + case CodeRange::JitEntry: + callerPC_ = nullptr; + callerFP_ = fp->rawCaller(); + break; + case CodeRange::Function: + fp = fp->wasmCaller(); + callerPC_ = fp->returnAddress(); + callerFP_ = fp->rawCaller(); + AssertMatchesCallSite(callerPC_, callerFP_); + break; + case CodeRange::ImportJitExit: + case CodeRange::ImportInterpExit: + case CodeRange::BuiltinThunk: + case CodeRange::TrapExit: + case CodeRange::DebugTrap: + case CodeRange::Throw: + case CodeRange::FarJumpIsland: + MOZ_CRASH("Unexpected CodeRange kind"); + } + + MOZ_ASSERT(!done()); +} + +static bool isSignatureCheckFail(uint32_t offsetInCode, + const CodeRange* codeRange) { + if (!codeRange->isFunction()) { + return false; + } + // checked call entry: 1. push Frame + // 2. set FP + // 3. signature check <--- check if we are here. + // 4. jump 7 + // unchecked call entry: 5. push Frame + // 6. set FP + // 7. function's code + return offsetInCode < codeRange->funcUncheckedCallEntry() && + (offsetInCode - codeRange->funcCheckedCallEntry()) > SetFP; +} + +const Instance* js::wasm::GetNearestEffectiveInstance(const Frame* fp) { + while (true) { + uint8_t* returnAddress = fp->returnAddress(); + const CodeRange* codeRange = nullptr; + const Code* code = LookupCode(returnAddress, &codeRange); + + if (!code) { + // It is a direct call from JIT. + AssertDirectJitCall(fp->jitEntryCaller()); + return ExtractCalleeInstanceFromFrameWithInstances(fp); + } + + MOZ_ASSERT(codeRange); + + if (codeRange->isEntry()) { + return ExtractCalleeInstanceFromFrameWithInstances(fp); + } + + MOZ_ASSERT(codeRange->kind() == CodeRange::Function); + MOZ_ASSERT(code); + const CallSite* callsite = code->lookupCallSite(returnAddress); + if (callsite->mightBeCrossInstance()) { + return ExtractCalleeInstanceFromFrameWithInstances(fp); + } + + fp = fp->wasmCaller(); + } +} + +Instance* js::wasm::GetNearestEffectiveInstance(Frame* fp) { + return const_cast( + GetNearestEffectiveInstance(const_cast(fp))); +} + +bool js::wasm::StartUnwinding(const RegisterState& registers, + UnwindState* unwindState, bool* unwoundCaller) { + // Shorthands. + uint8_t* const pc = (uint8_t*)registers.pc; + void** const sp = (void**)registers.sp; + + // The frame pointer might be: + // - in the process of tagging/untagging when calling into C++ code (this + // happens in wasm::SetExitFP); make sure it's untagged. + // - unreliable if it's not been set yet, in prologues. + uint8_t* fp = Frame::isExitFP(registers.fp) + ? Frame::untagExitFP(registers.fp) + : reinterpret_cast(registers.fp); + + // Get the CodeRange describing pc and the base address to which the + // CodeRange is relative. If the pc is not in a wasm module or a builtin + // thunk, then execution must be entering from or leaving to the C++ caller + // that pushed the JitActivation. + const CodeRange* codeRange; + uint8_t* codeBase; + const Code* code = nullptr; + + const CodeSegment* codeSegment = LookupCodeSegment(pc, &codeRange); + if (codeSegment) { + code = &codeSegment->code(); + codeBase = codeSegment->base(); + MOZ_ASSERT(codeRange); + } else if (!LookupBuiltinThunk(pc, &codeRange, &codeBase)) { + return false; + } + + // When the pc is inside the prologue/epilogue, the innermost call's Frame + // is not complete and thus fp points to the second-to-innermost call's + // Frame. Since fp can only tell you about its caller, naively unwinding + // while pc is in the prologue/epilogue would skip the second-to-innermost + // call. To avoid this problem, we use the static structure of the code in + // the prologue and epilogue to do the Right Thing. + uint32_t offsetInCode = pc - codeBase; + MOZ_ASSERT(offsetInCode >= codeRange->begin()); + MOZ_ASSERT(offsetInCode < codeRange->end()); + + // Compute the offset of the pc from the (unchecked call) entry of the code + // range. The checked call entry and the unchecked call entry have common + // prefix, so pc before signature check in the checked call entry is + // equivalent to the pc of the unchecked-call-entry. Thus, we can simplify the + // below case analysis by redirecting all pc-in-checked-call-entry before + // signature check cases to the pc-at-unchecked-call-entry case. + uint32_t offsetFromEntry; + if (codeRange->isFunction()) { + if (offsetInCode < codeRange->funcUncheckedCallEntry()) { + offsetFromEntry = offsetInCode - codeRange->funcCheckedCallEntry(); + } else { + offsetFromEntry = offsetInCode - codeRange->funcUncheckedCallEntry(); + } + } else { + offsetFromEntry = offsetInCode - codeRange->begin(); + } + + // Most cases end up unwinding to the caller state; not unwinding is the + // exception here. + *unwoundCaller = true; + + uint8_t* fixedFP = nullptr; + void* fixedPC = nullptr; + switch (codeRange->kind()) { + case CodeRange::Function: + case CodeRange::FarJumpIsland: + case CodeRange::ImportJitExit: + case CodeRange::ImportInterpExit: + case CodeRange::BuiltinThunk: + case CodeRange::DebugTrap: +#if defined(JS_CODEGEN_MIPS64) + if (codeRange->isThunk()) { + // The FarJumpIsland sequence temporary scrambles ra. + // Don't unwind to caller. + fixedPC = pc; + fixedFP = fp; + *unwoundCaller = false; + AssertMatchesCallSite( + Frame::fromUntaggedWasmExitFP(fp)->returnAddress(), + Frame::fromUntaggedWasmExitFP(fp)->rawCaller()); + } else if (offsetFromEntry < PushedFP) { + // On MIPS we rely on register state instead of state saved on + // stack until the wasm::Frame is completely built. + // On entry the return address is in ra (registers.lr) and + // fp holds the caller's fp. + fixedPC = (uint8_t*)registers.lr; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else +#elif defined(JS_CODEGEN_LOONG64) + if (codeRange->isThunk()) { + // The FarJumpIsland sequence temporary scrambles ra. + // Don't unwind to caller. + fixedPC = pc; + fixedFP = fp; + *unwoundCaller = false; + AssertMatchesCallSite( + Frame::fromUntaggedWasmExitFP(fp)->returnAddress(), + Frame::fromUntaggedWasmExitFP(fp)->rawCaller()); + } else if (offsetFromEntry < PushedFP) { + // On LoongArch we rely on register state instead of state saved on + // stack until the wasm::Frame is completely built. + // On entry the return address is in ra (registers.lr) and + // fp holds the caller's fp. + fixedPC = (uint8_t*)registers.lr; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else +#elif defined(JS_CODEGEN_RISCV64) + if (codeRange->isThunk()) { + // The FarJumpIsland sequence temporary scrambles ra. + // Don't unwind to caller. + fixedPC = pc; + fixedFP = fp; + *unwoundCaller = false; + AssertMatchesCallSite( + Frame::fromUntaggedWasmExitFP(fp)->returnAddress(), + Frame::fromUntaggedWasmExitFP(fp)->rawCaller()); + } else if (offsetFromEntry < PushedFP) { + // On Riscv64 we rely on register state instead of state saved on + // stack until the wasm::Frame is completely built. + // On entry the return address is in ra (registers.lr) and + // fp holds the caller's fp. + fixedPC = (uint8_t*)registers.lr; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else +#elif defined(JS_CODEGEN_ARM64) + if (offsetFromEntry < PushedFP || codeRange->isThunk()) { + // Constraints above ensure that this covers BeforePushRetAddr and + // PushedRetAddr. + // + // On ARM64 we subtract the size of the Frame from SP and then store + // values into the stack. Execution can be interrupted at various + // places in that sequence. We rely on the register state for our + // values. + fixedPC = (uint8_t*)registers.lr; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else +#elif defined(JS_CODEGEN_ARM) + if (offsetFromEntry == BeforePushRetAddr || codeRange->isThunk()) { + // The return address is still in lr and fp holds the caller's fp. + fixedPC = (uint8_t*)registers.lr; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else +#endif + if (offsetFromEntry == PushedRetAddr || codeRange->isThunk()) { + // The return address has been pushed on the stack but fp still + // points to the caller's fp. + fixedPC = sp[0]; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else if (offsetFromEntry == PushedFP) { + // The full Frame has been pushed; fp is still the caller's fp. + const auto* frame = Frame::fromUntaggedWasmExitFP(sp); + MOZ_ASSERT(frame->rawCaller() == fp); + fixedPC = frame->returnAddress(); + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); +#if defined(JS_CODEGEN_MIPS64) + } else if (offsetInCode >= codeRange->ret() - PoppedFP && + offsetInCode <= codeRange->ret()) { + // The fixedFP field of the Frame has been loaded into fp. + // The ra and instance might also be loaded, but the Frame structure is + // still on stack, so we can acess the ra form there. + MOZ_ASSERT(*sp == fp); + fixedPC = Frame::fromUntaggedWasmExitFP(sp)->returnAddress(); + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); +#elif defined(JS_CODEGEN_LOONG64) + } else if (offsetInCode >= codeRange->ret() - PoppedFP && + offsetInCode <= codeRange->ret()) { + // The fixedFP field of the Frame has been loaded into fp. + // The ra might also be loaded, but the Frame structure is still on + // stack, so we can acess the ra from there. + MOZ_ASSERT(*sp == fp); + fixedPC = Frame::fromUntaggedWasmExitFP(sp)->returnAddress(); + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); +#elif defined(JS_CODEGEN_RISCV64) + } else if (offsetInCode >= codeRange->ret() - PoppedFP && + offsetInCode <= codeRange->ret()) { + // The fixedFP field of the Frame has been loaded into fp. + // The ra might also be loaded, but the Frame structure is still on + // stack, so we can acess the ra from there. + MOZ_ASSERT(*sp == fp); + fixedPC = Frame::fromUntaggedWasmExitFP(sp)->returnAddress(); + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); +#elif defined(JS_CODEGEN_ARM64) + // The stack pointer does not move until all values have + // been restored so several cases can be coalesced here. + } else if (offsetInCode >= codeRange->ret() - PoppedFP && + offsetInCode <= codeRange->ret()) { + fixedPC = Frame::fromUntaggedWasmExitFP(sp)->returnAddress(); + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); +#else + } else if (offsetInCode >= codeRange->ret() - PoppedFP && + offsetInCode < codeRange->ret()) { + // The fixedFP field of the Frame has been popped into fp. + fixedPC = sp[1]; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); + } else if (offsetInCode == codeRange->ret()) { + // Both the instance and fixedFP fields have been popped and fp now + // points to the caller's frame. + fixedPC = sp[0]; + fixedFP = fp; + AssertMatchesCallSite(fixedPC, fixedFP); +#endif + } else { + if (isSignatureCheckFail(offsetInCode, codeRange)) { + // Frame has been pushed and FP has been set. + const auto* frame = Frame::fromUntaggedWasmExitFP(fp); + fixedFP = frame->rawCaller(); + fixedPC = frame->returnAddress(); + AssertMatchesCallSite(fixedPC, fixedFP); + break; + } + + // Not in the prologue/epilogue. + fixedPC = pc; + fixedFP = fp; + *unwoundCaller = false; + AssertMatchesCallSite( + Frame::fromUntaggedWasmExitFP(fp)->returnAddress(), + Frame::fromUntaggedWasmExitFP(fp)->rawCaller()); + break; + } + break; + case CodeRange::TrapExit: + // These code stubs execute after the prologue/epilogue have completed + // so pc/fp contains the right values here. + fixedPC = pc; + fixedFP = fp; + *unwoundCaller = false; + AssertMatchesCallSite(Frame::fromUntaggedWasmExitFP(fp)->returnAddress(), + Frame::fromUntaggedWasmExitFP(fp)->rawCaller()); + break; + case CodeRange::InterpEntry: + // The entry trampoline is the final frame in an wasm JitActivation. The + // entry trampoline also doesn't GeneratePrologue/Epilogue so we can't + // use the general unwinding logic above. + break; + case CodeRange::JitEntry: + // There's a jit frame above the current one; we don't care about pc + // since the Jit entry frame is a jit frame which can be considered as + // an exit frame. + if (offsetFromEntry < PushedFP) { + // We haven't pushed the jit caller's frame pointer yet, thus the jit + // frame is incomplete. During profiling frame iteration, it means that + // the jit profiling frame iterator won't be able to unwind this frame; + // drop it. + return false; + } + if (offsetInCode >= codeRange->ret() - PoppedFPJitEntry && + offsetInCode <= codeRange->ret()) { + // We've popped FP but still have to return. Similar to the + // |offsetFromEntry < PushedFP| case above, the JIT frame is now + // incomplete and we can't unwind. + return false; + } + // Set fixedFP to the address of the JitFrameLayout on the stack. + if (offsetFromEntry < SetFP) { + fixedFP = reinterpret_cast(sp); + } else { + fixedFP = fp; + } + fixedPC = nullptr; + break; + case CodeRange::Throw: + // The throw stub executes a small number of instructions before popping + // the entire activation. To simplify testing, we simply pretend throw + // stubs have already popped the entire stack. + return false; + } + + unwindState->code = code; + unwindState->codeRange = codeRange; + unwindState->fp = fixedFP; + unwindState->pc = fixedPC; + return true; +} + +ProfilingFrameIterator::ProfilingFrameIterator(const JitActivation& activation, + const RegisterState& state) + : code_(nullptr), + codeRange_(nullptr), + callerFP_(nullptr), + callerPC_(nullptr), + stackAddress_(nullptr), + unwoundJitCallerFP_(nullptr), + exitReason_(ExitReason::Fixed::None) { + // Let wasmExitFP take precedence to StartUnwinding when it is set since + // during the body of an exit stub, the register state may not be valid + // causing StartUnwinding() to abandon unwinding this activation. + if (activation.hasWasmExitFP()) { + exitReason_ = activation.wasmExitReason(); + initFromExitFP(activation.wasmExitFP()); + return; + } + + bool unwoundCaller; + UnwindState unwindState; + if (!StartUnwinding(state, &unwindState, &unwoundCaller)) { + MOZ_ASSERT(done()); + return; + } + + MOZ_ASSERT(unwindState.codeRange); + + if (unwoundCaller) { + callerFP_ = unwindState.fp; + callerPC_ = unwindState.pc; + } else { + callerFP_ = Frame::fromUntaggedWasmExitFP(unwindState.fp)->rawCaller(); + callerPC_ = Frame::fromUntaggedWasmExitFP(unwindState.fp)->returnAddress(); + } + + code_ = unwindState.code; + codeRange_ = unwindState.codeRange; + stackAddress_ = state.sp; + endStackAddress_ = state.sp; + MOZ_ASSERT(!done()); +} + +void ProfilingFrameIterator::operator++() { + MOZ_ASSERT(!done()); + MOZ_ASSERT(!unwoundJitCallerFP_); + + if (!exitReason_.isNone()) { + exitReason_ = ExitReason::None(); + MOZ_ASSERT(codeRange_); + MOZ_ASSERT(!done()); + return; + } + + if (codeRange_->isInterpEntry()) { + codeRange_ = nullptr; + MOZ_ASSERT(done()); + return; + } + + if (codeRange_->isJitEntry()) { + MOZ_ASSERT(callerFP_); + unwoundJitCallerFP_ = callerFP_; + callerPC_ = nullptr; + callerFP_ = nullptr; + codeRange_ = nullptr; + MOZ_ASSERT(done()); + return; + } + + MOZ_RELEASE_ASSERT(callerPC_); + + code_ = LookupCode(callerPC_, &codeRange_); + + if (!code_) { + // The parent frame is an inlined wasm call, callerFP_ points to the fake + // exit frame. + MOZ_ASSERT(!codeRange_); + AssertDirectJitCall(callerFP_); + unwoundJitCallerFP_ = callerFP_; + MOZ_ASSERT(done()); + return; + } + + MOZ_ASSERT(codeRange_); + + if (codeRange_->isInterpEntry()) { + callerPC_ = nullptr; + callerFP_ = nullptr; + MOZ_ASSERT(!done()); + return; + } + + if (codeRange_->isJitEntry()) { + MOZ_ASSERT(!done()); + return; + } + + MOZ_ASSERT(code_ == &GetNearestEffectiveInstance( + Frame::fromUntaggedWasmExitFP(callerFP_)) + ->code()); + + switch (codeRange_->kind()) { + case CodeRange::Function: + case CodeRange::ImportJitExit: + case CodeRange::ImportInterpExit: + case CodeRange::BuiltinThunk: + case CodeRange::TrapExit: + case CodeRange::DebugTrap: + case CodeRange::FarJumpIsland: { + stackAddress_ = callerFP_; + const auto* frame = Frame::fromUntaggedWasmExitFP(callerFP_); + callerPC_ = frame->returnAddress(); + AssertMatchesCallSite(callerPC_, frame->rawCaller()); + callerFP_ = frame->rawCaller(); + break; + } + case CodeRange::InterpEntry: + case CodeRange::JitEntry: + MOZ_CRASH("should have been guarded above"); + case CodeRange::Throw: + MOZ_CRASH("code range doesn't have frame"); + } + + MOZ_ASSERT(!done()); +} + +static const char* ThunkedNativeToDescription(SymbolicAddress func) { + MOZ_ASSERT(NeedsBuiltinThunk(func)); + switch (func) { + case SymbolicAddress::HandleDebugTrap: + case SymbolicAddress::HandleThrow: + case SymbolicAddress::HandleTrap: + case SymbolicAddress::CallImport_General: + case SymbolicAddress::CoerceInPlace_ToInt32: + case SymbolicAddress::CoerceInPlace_ToNumber: + case SymbolicAddress::CoerceInPlace_ToBigInt: + case SymbolicAddress::BoxValue_Anyref: + MOZ_ASSERT(!NeedsBuiltinThunk(func), + "not in sync with NeedsBuiltinThunk"); + break; + case SymbolicAddress::ToInt32: + return "call to asm.js native ToInt32 coercion (in wasm)"; + case SymbolicAddress::DivI64: + return "call to native i64.div_s (in wasm)"; + case SymbolicAddress::UDivI64: + return "call to native i64.div_u (in wasm)"; + case SymbolicAddress::ModI64: + return "call to native i64.rem_s (in wasm)"; + case SymbolicAddress::UModI64: + return "call to native i64.rem_u (in wasm)"; + case SymbolicAddress::TruncateDoubleToUint64: + return "call to native i64.trunc_u/f64 (in wasm)"; + case SymbolicAddress::TruncateDoubleToInt64: + return "call to native i64.trunc_s/f64 (in wasm)"; + case SymbolicAddress::SaturatingTruncateDoubleToUint64: + return "call to native i64.trunc_u:sat/f64 (in wasm)"; + case SymbolicAddress::SaturatingTruncateDoubleToInt64: + return "call to native i64.trunc_s:sat/f64 (in wasm)"; + case SymbolicAddress::Uint64ToDouble: + return "call to native f64.convert_u/i64 (in wasm)"; + case SymbolicAddress::Uint64ToFloat32: + return "call to native f32.convert_u/i64 (in wasm)"; + case SymbolicAddress::Int64ToDouble: + return "call to native f64.convert_s/i64 (in wasm)"; + case SymbolicAddress::Int64ToFloat32: + return "call to native f32.convert_s/i64 (in wasm)"; +#if defined(JS_CODEGEN_ARM) + case SymbolicAddress::aeabi_idivmod: + return "call to native i32.div_s (in wasm)"; + case SymbolicAddress::aeabi_uidivmod: + return "call to native i32.div_u (in wasm)"; +#endif + case SymbolicAddress::AllocateBigInt: + return "call to native newCell (in wasm)"; + case SymbolicAddress::ModD: + return "call to asm.js native f64 % (mod)"; + case SymbolicAddress::SinNativeD: + return "call to asm.js native f64 Math.sin"; + case SymbolicAddress::SinFdlibmD: + return "call to asm.js fdlibm f64 Math.sin"; + case SymbolicAddress::CosNativeD: + return "call to asm.js native f64 Math.cos"; + case SymbolicAddress::CosFdlibmD: + return "call to asm.js fdlibm f64 Math.cos"; + case SymbolicAddress::TanNativeD: + return "call to asm.js native f64 Math.tan"; + case SymbolicAddress::TanFdlibmD: + return "call to asm.js fdlibm f64 Math.tan"; + case SymbolicAddress::ASinD: + return "call to asm.js native f64 Math.asin"; + case SymbolicAddress::ACosD: + return "call to asm.js native f64 Math.acos"; + case SymbolicAddress::ATanD: + return "call to asm.js native f64 Math.atan"; + case SymbolicAddress::CeilD: + return "call to native f64.ceil (in wasm)"; + case SymbolicAddress::CeilF: + return "call to native f32.ceil (in wasm)"; + case SymbolicAddress::FloorD: + return "call to native f64.floor (in wasm)"; + case SymbolicAddress::FloorF: + return "call to native f32.floor (in wasm)"; + case SymbolicAddress::TruncD: + return "call to native f64.trunc (in wasm)"; + case SymbolicAddress::TruncF: + return "call to native f32.trunc (in wasm)"; + case SymbolicAddress::NearbyIntD: + return "call to native f64.nearest (in wasm)"; + case SymbolicAddress::NearbyIntF: + return "call to native f32.nearest (in wasm)"; + case SymbolicAddress::ExpD: + return "call to asm.js native f64 Math.exp"; + case SymbolicAddress::LogD: + return "call to asm.js native f64 Math.log"; + case SymbolicAddress::PowD: + return "call to asm.js native f64 Math.pow"; + case SymbolicAddress::ATan2D: + return "call to asm.js native f64 Math.atan2"; + case SymbolicAddress::MemoryGrowM32: + return "call to native memory.grow m32 (in wasm)"; + case SymbolicAddress::MemoryGrowM64: + return "call to native memory.grow m64 (in wasm)"; + case SymbolicAddress::MemorySizeM32: + return "call to native memory.size m32 (in wasm)"; + case SymbolicAddress::MemorySizeM64: + return "call to native memory.size m64 (in wasm)"; + case SymbolicAddress::WaitI32M32: + return "call to native i32.wait m32 (in wasm)"; + case SymbolicAddress::WaitI32M64: + return "call to native i32.wait m64 (in wasm)"; + case SymbolicAddress::WaitI64M32: + return "call to native i64.wait m32 (in wasm)"; + case SymbolicAddress::WaitI64M64: + return "call to native i64.wait m64 (in wasm)"; + case SymbolicAddress::WakeM32: + return "call to native wake m32 (in wasm)"; + case SymbolicAddress::WakeM64: + return "call to native wake m64 (in wasm)"; + case SymbolicAddress::CoerceInPlace_JitEntry: + return "out-of-line coercion for jit entry arguments (in wasm)"; + case SymbolicAddress::ReportV128JSCall: + return "jit call to v128 wasm function"; + case SymbolicAddress::MemCopyM32: + case SymbolicAddress::MemCopySharedM32: + return "call to native memory.copy m32 function"; + case SymbolicAddress::MemCopyM64: + case SymbolicAddress::MemCopySharedM64: + return "call to native memory.copy m64 function"; + case SymbolicAddress::DataDrop: + return "call to native data.drop function"; + case SymbolicAddress::MemFillM32: + case SymbolicAddress::MemFillSharedM32: + return "call to native memory.fill m32 function"; + case SymbolicAddress::MemFillM64: + case SymbolicAddress::MemFillSharedM64: + return "call to native memory.fill m64 function"; + case SymbolicAddress::MemInitM32: + return "call to native memory.init m32 function"; + case SymbolicAddress::MemInitM64: + return "call to native memory.init m64 function"; + case SymbolicAddress::TableCopy: + return "call to native table.copy function"; + case SymbolicAddress::TableFill: + return "call to native table.fill function"; + case SymbolicAddress::MemDiscardM32: + case SymbolicAddress::MemDiscardSharedM32: + return "call to native memory.discard m32 function"; + case SymbolicAddress::MemDiscardM64: + case SymbolicAddress::MemDiscardSharedM64: + return "call to native memory.discard m64 function"; + case SymbolicAddress::ElemDrop: + return "call to native elem.drop function"; + case SymbolicAddress::TableGet: + return "call to native table.get function"; + case SymbolicAddress::TableGrow: + return "call to native table.grow function"; + case SymbolicAddress::TableInit: + return "call to native table.init function"; + case SymbolicAddress::TableSet: + return "call to native table.set function"; + case SymbolicAddress::TableSize: + return "call to native table.size function"; + case SymbolicAddress::RefFunc: + return "call to native ref.func function"; + case SymbolicAddress::PostBarrier: + case SymbolicAddress::PostBarrierPrecise: + case SymbolicAddress::PostBarrierPreciseWithOffset: + return "call to native GC postbarrier (in wasm)"; + case SymbolicAddress::ExceptionNew: + return "call to native exception new (in wasm)"; + case SymbolicAddress::ThrowException: + return "call to native throw exception (in wasm)"; + case SymbolicAddress::StructNew: + case SymbolicAddress::StructNewUninit: + return "call to native struct.new (in wasm)"; + case SymbolicAddress::ArrayNew: + case SymbolicAddress::ArrayNewUninit: + return "call to native array.new (in wasm)"; + case SymbolicAddress::ArrayNewData: + return "call to native array.new_data function"; + case SymbolicAddress::ArrayNewElem: + return "call to native array.new_elem function"; + case SymbolicAddress::ArrayCopy: + return "call to native array.copy function"; +#define OP(op, export, sa_name, abitype, entry, idx) \ + case SymbolicAddress::sa_name: \ + return "call to native " #op " intrinsic (in wasm)"; + FOR_EACH_INTRINSIC(OP) +#undef OP +#ifdef WASM_CODEGEN_DEBUG + case SymbolicAddress::PrintI32: + case SymbolicAddress::PrintPtr: + case SymbolicAddress::PrintF32: + case SymbolicAddress::PrintF64: + case SymbolicAddress::PrintText: +#endif + case SymbolicAddress::Limit: + break; + } + return "?"; +} + +const char* ProfilingFrameIterator::label() const { + MOZ_ASSERT(!done()); + + // Use the same string for both time inside and under so that the two + // entries will be coalesced by the profiler. + // Must be kept in sync with /tools/profiler/tests/test_asm.js + static const char importJitDescription[] = "fast exit trampoline (in wasm)"; + static const char importInterpDescription[] = + "slow exit trampoline (in wasm)"; + static const char builtinNativeDescription[] = + "fast exit trampoline to native (in wasm)"; + static const char trapDescription[] = "trap handling (in wasm)"; + static const char debugTrapDescription[] = "debug trap handling (in wasm)"; + + if (!exitReason_.isFixed()) { + return ThunkedNativeToDescription(exitReason_.symbolic()); + } + + switch (exitReason_.fixed()) { + case ExitReason::Fixed::None: + break; + case ExitReason::Fixed::ImportJit: + return importJitDescription; + case ExitReason::Fixed::ImportInterp: + return importInterpDescription; + case ExitReason::Fixed::BuiltinNative: + return builtinNativeDescription; + case ExitReason::Fixed::Trap: + return trapDescription; + case ExitReason::Fixed::DebugTrap: + return debugTrapDescription; + } + + switch (codeRange_->kind()) { + case CodeRange::Function: + return code_->profilingLabel(codeRange_->funcIndex()); + case CodeRange::InterpEntry: + return "slow entry trampoline (in wasm)"; + case CodeRange::JitEntry: + return "fast entry trampoline (in wasm)"; + case CodeRange::ImportJitExit: + return importJitDescription; + case CodeRange::BuiltinThunk: + return builtinNativeDescription; + case CodeRange::ImportInterpExit: + return importInterpDescription; + case CodeRange::TrapExit: + return trapDescription; + case CodeRange::DebugTrap: + return debugTrapDescription; + case CodeRange::FarJumpIsland: + return "interstitial (in wasm)"; + case CodeRange::Throw: + MOZ_CRASH("does not have a frame"); + } + + MOZ_CRASH("bad code range kind"); +} diff --git a/js/src/wasm/WasmFrameIter.h b/js/src/wasm/WasmFrameIter.h new file mode 100644 index 0000000000..cec227a43f --- /dev/null +++ b/js/src/wasm/WasmFrameIter.h @@ -0,0 +1,277 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2014 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_frame_iter_h +#define wasm_frame_iter_h + +#include "js/ProfilingFrameIterator.h" +#include "js/TypeDecls.h" + +namespace js { + +namespace jit { +class JitActivation; +class MacroAssembler; +struct Register; +enum class FrameType; +} // namespace jit + +namespace wasm { + +class CallIndirectId; +class Code; +class CodeRange; +class DebugFrame; +class Instance; +class Instance; + +struct CallableOffsets; +struct FuncOffsets; +struct Offsets; +class Frame; + +using RegisterState = JS::ProfilingFrameIterator::RegisterState; + +// Iterates over a linear group of wasm frames of a single wasm JitActivation, +// called synchronously from C++ in the wasm thread. It will stop at the first +// frame that is not of the same kind, or at the end of an activation. +// +// If you want to handle every kind of frames (including JS jit frames), use +// JitFrameIter. + +class WasmFrameIter { + public: + enum class Unwind { True, False }; + static constexpr uint32_t ColumnBit = 1u << 31; + + private: + jit::JitActivation* activation_; + const Code* code_; + const CodeRange* codeRange_; + unsigned lineOrBytecode_; + Frame* fp_; + Instance* instance_; + uint8_t* unwoundCallerFP_; + mozilla::Maybe unwoundJitFrameType_; + Unwind unwind_; + void** unwoundAddressOfReturnAddress_; + uint8_t* resumePCinCurrentFrame_; + + void popFrame(); + + public: + // See comment above this class definition. + explicit WasmFrameIter(jit::JitActivation* activation, Frame* fp = nullptr); + const jit::JitActivation* activation() const { return activation_; } + void setUnwind(Unwind unwind) { unwind_ = unwind; } + void operator++(); + bool done() const; + const char* filename() const; + const char16_t* displayURL() const; + bool mutedErrors() const; + JSAtom* functionDisplayAtom() const; + unsigned lineOrBytecode() const; + uint32_t funcIndex() const; + unsigned computeLine(uint32_t* column) const; + const CodeRange* codeRange() const { return codeRange_; } + void** unwoundAddressOfReturnAddress() const; + bool debugEnabled() const; + DebugFrame* debugFrame() const; + jit::FrameType unwoundJitFrameType() const; + bool hasUnwoundJitFrame() const; + uint8_t* unwoundCallerFP() const { return unwoundCallerFP_; } + Frame* frame() const { return fp_; } + Instance* instance() const { return instance_; } + + // Returns the address of the next instruction that will execute in this + // frame, once control returns to this frame. + uint8_t* resumePCinCurrentFrame() const; +}; + +enum class SymbolicAddress; + +// An ExitReason describes the possible reasons for leaving compiled wasm +// code or the state of not having left compiled wasm code +// (ExitReason::None). It is either a known reason, or a enumeration to a native +// function that is used for better display in the profiler. +class ExitReason { + public: + enum class Fixed : uint32_t { + None, // default state, the pc is in wasm code + ImportJit, // fast-path call directly into JIT code + ImportInterp, // slow-path call into C++ Invoke() + BuiltinNative, // fast-path call directly into native C++ code + Trap, // call to trap handler + DebugTrap // call to debug trap handler + }; + + private: + uint32_t payload_; + + ExitReason() : ExitReason(Fixed::None) {} + + public: + MOZ_IMPLICIT ExitReason(Fixed exitReason) + : payload_(0x0 | (uint32_t(exitReason) << 1)) { + MOZ_ASSERT(isFixed()); + MOZ_ASSERT_IF(isNone(), payload_ == 0); + } + + explicit ExitReason(SymbolicAddress sym) + : payload_(0x1 | (uint32_t(sym) << 1)) { + MOZ_ASSERT(uint32_t(sym) <= (UINT32_MAX << 1), "packing constraints"); + MOZ_ASSERT(!isFixed()); + } + + static ExitReason Decode(uint32_t payload) { + ExitReason reason; + reason.payload_ = payload; + return reason; + } + + static ExitReason None() { return ExitReason(ExitReason::Fixed::None); } + + bool isFixed() const { return (payload_ & 0x1) == 0; } + bool isNone() const { return isFixed() && fixed() == Fixed::None; } + bool isNative() const { + return !isFixed() || fixed() == Fixed::BuiltinNative; + } + + uint32_t encode() const { return payload_; } + Fixed fixed() const { + MOZ_ASSERT(isFixed()); + return Fixed(payload_ >> 1); + } + SymbolicAddress symbolic() const { + MOZ_ASSERT(!isFixed()); + return SymbolicAddress(payload_ >> 1); + } +}; + +// Iterates over the frames of a single wasm JitActivation, given an +// asynchronously-profiled thread's state. +class ProfilingFrameIterator { + const Code* code_; + const CodeRange* codeRange_; + uint8_t* callerFP_; + void* callerPC_; + void* stackAddress_; + // See JS::ProfilingFrameIterator::endStackAddress_ comment. + void* endStackAddress_ = nullptr; + uint8_t* unwoundJitCallerFP_; + ExitReason exitReason_; + + void initFromExitFP(const Frame* fp); + + public: + ProfilingFrameIterator(); + + // Start unwinding at a non-innermost activation that has necessarily been + // exited from wasm code (and thus activation.hasWasmExitFP). + explicit ProfilingFrameIterator(const jit::JitActivation& activation); + + // Start unwinding at a group of wasm frames after unwinding an inner group + // of JSJit frames. + explicit ProfilingFrameIterator(const Frame* fp); + + // Start unwinding at the innermost activation given the register state when + // the thread was suspended. + ProfilingFrameIterator(const jit::JitActivation& activation, + const RegisterState& state); + + void operator++(); + + bool done() const { + MOZ_ASSERT_IF(!exitReason_.isNone(), codeRange_); + return !codeRange_; + } + + void* stackAddress() const { + MOZ_ASSERT(!done()); + return stackAddress_; + } + uint8_t* unwoundJitCallerFP() const { + MOZ_ASSERT(done()); + return unwoundJitCallerFP_; + } + const char* label() const; + + void* endStackAddress() const { return endStackAddress_; } +}; + +// Prologue/epilogue code generation + +void SetExitFP(jit::MacroAssembler& masm, ExitReason reason, + jit::Register scratch); +void ClearExitFP(jit::MacroAssembler& masm, jit::Register scratch); + +void GenerateExitPrologue(jit::MacroAssembler& masm, unsigned framePushed, + ExitReason reason, CallableOffsets* offsets); +void GenerateExitEpilogue(jit::MacroAssembler& masm, unsigned framePushed, + ExitReason reason, CallableOffsets* offsets); + +void GenerateJitExitPrologue(jit::MacroAssembler& masm, unsigned framePushed, + CallableOffsets* offsets); +void GenerateJitExitEpilogue(jit::MacroAssembler& masm, unsigned framePushed, + CallableOffsets* offsets); + +void GenerateJitEntryPrologue(jit::MacroAssembler& masm, + CallableOffsets* offsets); +void GenerateJitEntryEpilogue(jit::MacroAssembler& masm, + CallableOffsets* offsets); + +void GenerateFunctionPrologue(jit::MacroAssembler& masm, + const CallIndirectId& callIndirectId, + const mozilla::Maybe& tier1FuncIndex, + FuncOffsets* offsets); +void GenerateFunctionEpilogue(jit::MacroAssembler& masm, unsigned framePushed, + FuncOffsets* offsets); + +// Iterates through frames for either possible cross-instance call or an entry +// stub to obtain instance that corresponds to the passed fp. +const Instance* GetNearestEffectiveInstance(const Frame* fp); +Instance* GetNearestEffectiveInstance(Frame* fp); + +// Describes register state and associated code at a given call frame. + +struct UnwindState { + uint8_t* fp; + void* pc; + const Code* code; + const CodeRange* codeRange; + UnwindState() : fp(nullptr), pc(nullptr), code(nullptr), codeRange(nullptr) {} +}; + +// Ensures the register state at a call site is consistent: pc must be in the +// code range of the code described by fp. This prevents issues when using +// the values of pc/fp, especially at call sites boundaries, where the state +// hasn't fully transitioned from the caller's to the callee's. +// +// unwoundCaller is set to true if we were in a transitional state and had to +// rewind to the caller's frame instead of the current frame. +// +// Returns true if it was possible to get to a clear state, or false if the +// frame should be ignored. + +bool StartUnwinding(const RegisterState& registers, UnwindState* unwindState, + bool* unwoundCaller); + +} // namespace wasm +} // namespace js + +#endif // wasm_frame_iter_h diff --git a/js/src/wasm/WasmGC.cpp b/js/src/wasm/WasmGC.cpp new file mode 100644 index 0000000000..60ca38ebc1 --- /dev/null +++ b/js/src/wasm/WasmGC.cpp @@ -0,0 +1,314 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2019 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmGC.h" +#include "wasm/WasmInstance.h" +#include "jit/MacroAssembler-inl.h" + +using mozilla::DebugOnly; + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +wasm::StackMap* wasm::ConvertStackMapBoolVectorToStackMap( + const StackMapBoolVector& vec, bool hasRefs) { + wasm::StackMap* stackMap = wasm::StackMap::create(vec.length()); + if (!stackMap) { + return nullptr; + } + + bool hasRefsObserved = false; + size_t i = 0; + for (bool b : vec) { + if (b) { + stackMap->setBit(i); + hasRefsObserved = true; + } + i++; + } + MOZ_RELEASE_ASSERT(hasRefs == hasRefsObserved); + + return stackMap; +} + +// Generate a stackmap for a function's stack-overflow-at-entry trap, with +// the structure: +// +// +// | ++ +// | ++ +// | ++ +// | | +// Lowest Addr Highest Addr +// +// The caller owns the resulting stackmap. This assumes a grow-down stack. +// +// For non-debug builds, if the stackmap would contain no pointers, no +// stackmap is created, and nullptr is returned. For a debug build, a +// stackmap is always created and returned. +// +// The "space reserved before trap" is the space reserved by +// MacroAssembler::wasmReserveStackChecked, in the case where the frame is +// "small", as determined by that function. +bool wasm::CreateStackMapForFunctionEntryTrap( + const wasm::ArgTypeVector& argTypes, const RegisterOffsets& trapExitLayout, + size_t trapExitLayoutWords, size_t nBytesReservedBeforeTrap, + size_t nInboundStackArgBytes, wasm::StackMap** result) { + // Ensure this is defined on all return paths. + *result = nullptr; + + // The size of the wasm::Frame itself. + const size_t nFrameBytes = sizeof(wasm::Frame); + + // The size of the register dump (trap) area. + const size_t trapExitLayoutBytes = trapExitLayoutWords * sizeof(void*); + + // This is the total number of bytes covered by the map. + const DebugOnly nTotalBytes = trapExitLayoutBytes + + nBytesReservedBeforeTrap + nFrameBytes + + nInboundStackArgBytes; + + // Create the stackmap initially in this vector. Since most frames will + // contain 128 or fewer words, heap allocation is avoided in the majority of + // cases. vec[0] is for the lowest address in the map, vec[N-1] is for the + // highest address in the map. + StackMapBoolVector vec; + + // Keep track of whether we've actually seen any refs. + bool hasRefs = false; + + // REG DUMP AREA + wasm::ExitStubMapVector trapExitExtras; + if (!GenerateStackmapEntriesForTrapExit( + argTypes, trapExitLayout, trapExitLayoutWords, &trapExitExtras)) { + return false; + } + MOZ_ASSERT(trapExitExtras.length() == trapExitLayoutWords); + + if (!vec.appendN(false, trapExitLayoutWords)) { + return false; + } + for (size_t i = 0; i < trapExitLayoutWords; i++) { + vec[i] = trapExitExtras[i]; + hasRefs |= vec[i]; + } + + // SPACE RESERVED BEFORE TRAP + MOZ_ASSERT(nBytesReservedBeforeTrap % sizeof(void*) == 0); + if (!vec.appendN(false, nBytesReservedBeforeTrap / sizeof(void*))) { + return false; + } + + // SPACE FOR FRAME + if (!vec.appendN(false, nFrameBytes / sizeof(void*))) { + return false; + } + + // INBOUND ARG AREA + MOZ_ASSERT(nInboundStackArgBytes % sizeof(void*) == 0); + const size_t numStackArgWords = nInboundStackArgBytes / sizeof(void*); + + const size_t wordsSoFar = vec.length(); + if (!vec.appendN(false, numStackArgWords)) { + return false; + } + + for (WasmABIArgIter i(argTypes); !i.done(); i++) { + ABIArg argLoc = *i; + if (argLoc.kind() == ABIArg::Stack && + argTypes[i.index()] == MIRType::RefOrNull) { + uint32_t offset = argLoc.offsetFromArgBase(); + MOZ_ASSERT(offset < nInboundStackArgBytes); + MOZ_ASSERT(offset % sizeof(void*) == 0); + vec[wordsSoFar + offset / sizeof(void*)] = true; + hasRefs = true; + } + } + +#ifndef DEBUG + // We saw no references, and this is a non-debug build, so don't bother + // building the stackmap. + if (!hasRefs) { + return true; + } +#endif + + // Convert vec into a wasm::StackMap. + MOZ_ASSERT(vec.length() * sizeof(void*) == nTotalBytes); + wasm::StackMap* stackMap = ConvertStackMapBoolVectorToStackMap(vec, hasRefs); + if (!stackMap) { + return false; + } + stackMap->setExitStubWords(trapExitLayoutWords); + + stackMap->setFrameOffsetFromTop(nFrameBytes / sizeof(void*) + + numStackArgWords); +#ifdef DEBUG + for (uint32_t i = 0; i < nFrameBytes / sizeof(void*); i++) { + MOZ_ASSERT(stackMap->getBit(stackMap->header.numMappedWords - + stackMap->header.frameOffsetFromTop + i) == 0); + } +#endif + + *result = stackMap; + return true; +} + +bool wasm::GenerateStackmapEntriesForTrapExit( + const ArgTypeVector& args, const RegisterOffsets& trapExitLayout, + const size_t trapExitLayoutNumWords, ExitStubMapVector* extras) { + MOZ_ASSERT(extras->empty()); + + if (!extras->appendN(false, trapExitLayoutNumWords)) { + return false; + } + + for (WasmABIArgIter i(args); !i.done(); i++) { + if (!i->argInRegister() || i.mirType() != MIRType::RefOrNull) { + continue; + } + + size_t offsetFromTop = trapExitLayout.getOffset(i->gpr()); + + // If this doesn't hold, the associated register wasn't saved by + // the trap exit stub. Better to crash now than much later, in + // some obscure place, and possibly with security consequences. + MOZ_RELEASE_ASSERT(offsetFromTop < trapExitLayoutNumWords); + + // offsetFromTop is an offset in words down from the highest + // address in the exit stub save area. Switch it around to be an + // offset up from the bottom of the (integer register) save area. + size_t offsetFromBottom = trapExitLayoutNumWords - 1 - offsetFromTop; + + (*extras)[offsetFromBottom] = true; + } + + return true; +} + +void wasm::EmitWasmPreBarrierGuard(MacroAssembler& masm, Register instance, + Register scratch, Register valueAddr, + size_t valueOffset, Label* skipBarrier, + BytecodeOffset* trapOffset) { + // If no incremental GC has started, we don't need the barrier. + masm.loadPtr( + Address(instance, Instance::offsetOfAddressOfNeedsIncrementalBarrier()), + scratch); + masm.branchTest32(Assembler::Zero, Address(scratch, 0), Imm32(0x1), + skipBarrier); + + // Emit metadata for a potential null access when reading the previous value. + if (trapOffset) { + masm.append(wasm::Trap::NullPointerDereference, + wasm::TrapSite(masm.currentOffset(), *trapOffset)); + } + + // If the previous value is null, we don't need the barrier. + masm.loadPtr(Address(valueAddr, valueOffset), scratch); + masm.branchTestPtr(Assembler::Zero, scratch, scratch, skipBarrier); +} + +void wasm::EmitWasmPreBarrierCall(MacroAssembler& masm, Register instance, + Register scratch, Register valueAddr, + size_t valueOffset) { + MOZ_ASSERT(valueAddr == PreBarrierReg); + + // Add the offset to the PreBarrierReg, if any. + if (valueOffset != 0) { + masm.addPtr(Imm32(valueOffset), valueAddr); + } + +#if defined(DEBUG) && defined(JS_CODEGEN_ARM64) + // The prebarrier assumes that x28 == sp. + Label ok; + masm.Cmp(sp, vixl::Operand(x28)); + masm.B(&ok, Assembler::Equal); + masm.breakpoint(); + masm.bind(&ok); +#endif + + // Load and call the pre-write barrier code. It will preserve all volatile + // registers. + masm.loadPtr(Address(instance, Instance::offsetOfPreBarrierCode()), scratch); + masm.call(scratch); + + // Remove the offset we folded into PreBarrierReg, if any. + if (valueOffset != 0) { + masm.subPtr(Imm32(valueOffset), valueAddr); + } +} + +void wasm::EmitWasmPostBarrierGuard(MacroAssembler& masm, + const Maybe& object, + Register otherScratch, Register setValue, + Label* skipBarrier) { + // If the pointer being stored is null, no barrier. + masm.branchTestPtr(Assembler::Zero, setValue, setValue, skipBarrier); + + // If there is a containing object and it is in the nursery, no barrier. + if (object) { + masm.branchPtrInNurseryChunk(Assembler::Equal, *object, otherScratch, + skipBarrier); + } + + // If the pointer being stored is to a tenured object, no barrier. + masm.branchPtrInNurseryChunk(Assembler::NotEqual, setValue, otherScratch, + skipBarrier); +} + +#ifdef DEBUG +bool wasm::IsValidStackMapKey(bool debugEnabled, const uint8_t* nextPC) { +# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) + const uint8_t* insn = nextPC; + return (insn[-2] == 0x0F && insn[-1] == 0x0B) || // ud2 + (insn[-2] == 0xFF && (insn[-1] & 0xF8) == 0xD0) || // call *%r_ + insn[-5] == 0xE8; // call simm32 + +# elif defined(JS_CODEGEN_ARM) + const uint32_t* insn = (const uint32_t*)nextPC; + return ((uintptr_t(insn) & 3) == 0) && // must be ARM, not Thumb + (insn[-1] == 0xe7f000f0 || // udf + (insn[-1] & 0xfffffff0) == 0xe12fff30 || // blx reg (ARM, enc A1) + (insn[-1] & 0x0f000000) == 0x0b000000); // bl.cc simm24 (ARM, enc A1) + +# elif defined(JS_CODEGEN_ARM64) + const uint32_t hltInsn = 0xd4a00000; + const uint32_t* insn = (const uint32_t*)nextPC; + return ((uintptr_t(insn) & 3) == 0) && + (insn[-1] == hltInsn || // hlt + (insn[-1] & 0xfffffc1f) == 0xd63f0000 || // blr reg + (insn[-1] & 0xfc000000) == 0x94000000); // bl simm26 + +# elif defined(JS_CODEGEN_MIPS64) + // TODO (bug 1699696): Implement this. As for the platforms above, we need to + // enumerate all code sequences that can precede the stackmap location. + return true; +# elif defined(JS_CODEGEN_LOONG64) + // TODO(loong64): Implement IsValidStackMapKey. + return true; +# elif defined(JS_CODEGEN_RISCV64) + const uint32_t* insn = (const uint32_t*)nextPC; + return (((uintptr_t(insn) & 3) == 0) && + (insn[-1] == 0x00006037 && insn[-2] == 0x00100073) || // break; + ((insn[-1] & kBaseOpcodeMask) == JALR)); +# else + MOZ_CRASH("IsValidStackMapKey: requires implementation on this platform"); +# endif +} +#endif diff --git a/js/src/wasm/WasmGC.h b/js/src/wasm/WasmGC.h new file mode 100644 index 0000000000..7a42a17988 --- /dev/null +++ b/js/src/wasm/WasmGC.h @@ -0,0 +1,495 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2019 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_gc_h +#define wasm_gc_h + +#include "mozilla/BinarySearch.h" + +#include "jit/ABIArgGenerator.h" // For ABIArgIter +#include "js/AllocPolicy.h" +#include "js/Vector.h" +#include "util/Memory.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmFrame.h" +#include "wasm/WasmSerialize.h" + +namespace js { + +namespace jit { +class Label; +class MacroAssembler; +} // namespace jit + +namespace wasm { + +class ArgTypeVector; +class BytecodeOffset; + +using jit::Label; +using jit::MIRType; +using jit::Register; + +// Definitions for stackmaps. + +using ExitStubMapVector = Vector; + +struct StackMapHeader { + explicit StackMapHeader(uint32_t numMappedWords = 0) + : numMappedWords(numMappedWords), + numExitStubWords(0), + frameOffsetFromTop(0), + hasDebugFrameWithLiveRefs(0) {} + + // The total number of stack words covered by the map .. + static constexpr size_t MappedWordsBits = 30; + uint32_t numMappedWords : MappedWordsBits; + + // .. of which this many are "exit stub" extras + static constexpr size_t ExitStubWordsBits = 6; + uint32_t numExitStubWords : ExitStubWordsBits; + + // Where is Frame* relative to the top? This is an offset in words. On every + // platform, FrameOffsetBits needs to be at least + // ceil(log2(MaxParams*sizeof-biggest-param-type-in-words)). The most + // constraining platforms are 32-bit with SIMD support, currently x86-32. + static constexpr size_t FrameOffsetBits = 12; + uint32_t frameOffsetFromTop : FrameOffsetBits; + + // Notes the presence of a DebugFrame with possibly-live references. A + // DebugFrame may or may not contain GC-managed data; in situations when it is + // possible that any pointers in the DebugFrame are non-null, the DebugFrame + // gets a stackmap. + uint32_t hasDebugFrameWithLiveRefs : 1; + + WASM_CHECK_CACHEABLE_POD(numMappedWords, numExitStubWords, frameOffsetFromTop, + hasDebugFrameWithLiveRefs); + + static constexpr uint32_t maxMappedWords = (1 << MappedWordsBits) - 1; + static constexpr uint32_t maxExitStubWords = (1 << ExitStubWordsBits) - 1; + static constexpr uint32_t maxFrameOffsetFromTop = (1 << FrameOffsetBits) - 1; + + static constexpr size_t MaxParamSize = + std::max(sizeof(jit::FloatRegisters::RegisterContent), + sizeof(jit::Registers::RegisterContent)); + + // Add 16 words to account for the size of FrameWithInstances including any + // shadow stack (at worst 8 words total), and then a little headroom in case + // the argument area had to be aligned. + static_assert(FrameWithInstances::sizeOf() / sizeof(void*) <= 8); + static_assert(maxFrameOffsetFromTop >= + (MaxParams * MaxParamSize / sizeof(void*)) + 16, + "limited size of the offset field"); +}; + +WASM_DECLARE_CACHEABLE_POD(StackMapHeader); + +// This is the expected size for the header +static_assert(sizeof(StackMapHeader) == 8, + "wasm::StackMapHeader has unexpected size"); + +// A StackMap is a bit-array containing numMappedWords bits, one bit per +// word of stack. Bit index zero is for the lowest addressed word in the +// range. +// +// This is a variable-length structure whose size must be known at creation +// time. +// +// Users of the map will know the address of the wasm::Frame that is covered +// by this map. In order that they can calculate the exact address range +// covered by the map, the map also stores the offset, from the highest +// addressed word of the map, of the embedded wasm::Frame. This is an offset +// down from the highest address, rather than up from the lowest, so as to +// limit its range to FrameOffsetBits bits. +// +// The stackmap may also cover a DebugFrame (all DebugFrames which may +// potentially contain live pointers into the JS heap get a map). If so that +// can be noted, since users of the map need to trace pointers in a +// DebugFrame. +// +// Finally, for sanity checking only, for stackmaps associated with a wasm +// trap exit stub, the number of words used by the trap exit stub save area +// is also noted. This is used in Instance::traceFrame to check that the +// TrapExitDummyValue is in the expected place in the frame. +struct StackMap final { + // The header contains the constant-sized fields before the variable-sized + // bitmap that follows. + StackMapHeader header; + + private: + // The variable-sized bitmap. + uint32_t bitmap[1]; + + explicit StackMap(uint32_t numMappedWords) : header(numMappedWords) { + const uint32_t nBitmap = calcNBitmap(header.numMappedWords); + memset(bitmap, 0, nBitmap * sizeof(bitmap[0])); + } + explicit StackMap(const StackMapHeader& header) : header(header) { + const uint32_t nBitmap = calcNBitmap(header.numMappedWords); + memset(bitmap, 0, nBitmap * sizeof(bitmap[0])); + } + + public: + static StackMap* create(uint32_t numMappedWords) { + size_t size = allocationSizeInBytes(numMappedWords); + char* buf = (char*)js_malloc(size); + if (!buf) { + return nullptr; + } + return ::new (buf) StackMap(numMappedWords); + } + static StackMap* create(const StackMapHeader& header) { + size_t size = allocationSizeInBytes(header.numMappedWords); + char* buf = (char*)js_malloc(size); + if (!buf) { + return nullptr; + } + return ::new (buf) StackMap(header); + } + + void destroy() { js_free((char*)this); } + + // Returns the size of a `StackMap` allocated with `numMappedWords`. + static size_t allocationSizeInBytes(uint32_t numMappedWords) { + uint32_t nBitmap = calcNBitmap(numMappedWords); + return sizeof(StackMap) + (nBitmap - 1) * sizeof(bitmap[0]); + } + + // Returns the allocated size of this `StackMap`. + size_t allocationSizeInBytes() const { + return allocationSizeInBytes(header.numMappedWords); + } + + // Record the number of words in the map used as a wasm trap exit stub + // save area. See comment above. + void setExitStubWords(uint32_t nWords) { + MOZ_ASSERT(header.numExitStubWords == 0); + MOZ_RELEASE_ASSERT(nWords <= header.maxExitStubWords); + MOZ_ASSERT(nWords <= header.numMappedWords); + header.numExitStubWords = nWords; + } + + // Record the offset from the highest-addressed word of the map, that the + // wasm::Frame lives at. See comment above. + void setFrameOffsetFromTop(uint32_t nWords) { + MOZ_ASSERT(header.frameOffsetFromTop == 0); + MOZ_RELEASE_ASSERT(nWords <= StackMapHeader::maxFrameOffsetFromTop); + MOZ_ASSERT(header.frameOffsetFromTop < header.numMappedWords); + header.frameOffsetFromTop = nWords; + } + + // If the frame described by this StackMap includes a DebugFrame, call here to + // record that fact. + void setHasDebugFrameWithLiveRefs() { + MOZ_ASSERT(header.hasDebugFrameWithLiveRefs == 0); + header.hasDebugFrameWithLiveRefs = 1; + } + + inline void setBit(uint32_t bitIndex) { + MOZ_ASSERT(bitIndex < header.numMappedWords); + uint32_t wordIndex = bitIndex / wordsPerBitmapElem; + uint32_t wordOffset = bitIndex % wordsPerBitmapElem; + bitmap[wordIndex] |= (1 << wordOffset); + } + + inline uint32_t getBit(uint32_t bitIndex) const { + MOZ_ASSERT(bitIndex < header.numMappedWords); + uint32_t wordIndex = bitIndex / wordsPerBitmapElem; + uint32_t wordOffset = bitIndex % wordsPerBitmapElem; + return (bitmap[wordIndex] >> wordOffset) & 1; + } + + inline uint8_t* rawBitmap() { return (uint8_t*)&bitmap; } + inline const uint8_t* rawBitmap() const { return (const uint8_t*)&bitmap; } + inline size_t rawBitmapLengthInBytes() const { + return calcNBitmap(header.numMappedWords) * sizeof(uint32_t); + } + + private: + static constexpr uint32_t wordsPerBitmapElem = sizeof(bitmap[0]) * 8; + + static uint32_t calcNBitmap(uint32_t numMappedWords) { + MOZ_RELEASE_ASSERT(numMappedWords <= StackMapHeader::maxMappedWords); + uint32_t nBitmap = + (numMappedWords + wordsPerBitmapElem - 1) / wordsPerBitmapElem; + return nBitmap == 0 ? 1 : nBitmap; + } +}; + +// This is the expected size for a map that covers 32 or fewer words. +static_assert(sizeof(StackMap) == 12, "wasm::StackMap has unexpected size"); + +class StackMaps { + public: + // A Maplet holds a single code-address-to-map binding. Note that the + // code address is the lowest address of the instruction immediately + // following the instruction of interest, not of the instruction of + // interest itself. In practice (at least for the Wasm Baseline compiler) + // this means that |nextInsnAddr| points either immediately after a call + // instruction, after a trap instruction or after a no-op. + struct Maplet { + const uint8_t* nextInsnAddr; + StackMap* map; + Maplet(const uint8_t* nextInsnAddr, StackMap* map) + : nextInsnAddr(nextInsnAddr), map(map) {} + void offsetBy(uintptr_t delta) { nextInsnAddr += delta; } + bool operator<(const Maplet& other) const { + return uintptr_t(nextInsnAddr) < uintptr_t(other.nextInsnAddr); + } + }; + + private: + bool sorted_; + Vector mapping_; + + public: + StackMaps() : sorted_(false) {} + ~StackMaps() { + for (auto& maplet : mapping_) { + maplet.map->destroy(); + maplet.map = nullptr; + } + } + [[nodiscard]] bool add(const uint8_t* nextInsnAddr, StackMap* map) { + MOZ_ASSERT(!sorted_); + return mapping_.append(Maplet(nextInsnAddr, map)); + } + [[nodiscard]] bool add(const Maplet& maplet) { + return add(maplet.nextInsnAddr, maplet.map); + } + void clear() { + for (auto& maplet : mapping_) { + maplet.nextInsnAddr = nullptr; + maplet.map = nullptr; + } + mapping_.clear(); + } + bool empty() const { return mapping_.empty(); } + size_t length() const { return mapping_.length(); } + Maplet* getRef(size_t i) { return &mapping_[i]; } + Maplet get(size_t i) const { return mapping_[i]; } + Maplet move(size_t i) { + Maplet m = mapping_[i]; + mapping_[i].map = nullptr; + return m; + } + void offsetBy(uintptr_t delta) { + for (auto& maplet : mapping_) maplet.offsetBy(delta); + } + void finishAndSort() { + MOZ_ASSERT(!sorted_); + std::sort(mapping_.begin(), mapping_.end()); + sorted_ = true; + } + void finishAlreadySorted() { + MOZ_ASSERT(!sorted_); + MOZ_ASSERT(std::is_sorted(mapping_.begin(), mapping_.end())); + sorted_ = true; + } + const StackMap* findMap(const uint8_t* nextInsnAddr) const { + struct Comparator { + int operator()(Maplet aVal) const { + if (uintptr_t(mTarget) < uintptr_t(aVal.nextInsnAddr)) { + return -1; + } + if (uintptr_t(mTarget) > uintptr_t(aVal.nextInsnAddr)) { + return 1; + } + return 0; + } + explicit Comparator(const uint8_t* aTarget) : mTarget(aTarget) {} + const uint8_t* mTarget; + }; + + size_t result; + if (mozilla::BinarySearchIf(mapping_, 0, mapping_.length(), + Comparator(nextInsnAddr), &result)) { + return mapping_[result].map; + } + + return nullptr; + } +}; + +// Supporting code for creation of stackmaps. + +// StackArgAreaSizeUnaligned returns the size, in bytes, of the stack arg area +// size needed to pass |argTypes|, excluding any alignment padding beyond the +// size of the area as a whole. The size is as determined by the platforms +// native ABI. +// +// StackArgAreaSizeAligned returns the same, but rounded up to the nearest 16 +// byte boundary. +// +// Note, StackArgAreaSize{Unaligned,Aligned}() must process all the arguments +// in order to take into account all necessary alignment constraints. The +// signature must include any receiver argument -- in other words, it must be +// the complete native-ABI-level call signature. +template +static inline size_t StackArgAreaSizeUnaligned(const T& argTypes) { + jit::WasmABIArgIter i(argTypes); + while (!i.done()) { + i++; + } + return i.stackBytesConsumedSoFar(); +} + +static inline size_t StackArgAreaSizeUnaligned( + const SymbolicAddressSignature& saSig) { + // WasmABIArgIter::ABIArgIter wants the items to be iterated over to be + // presented in some type that has methods length() and operator[]. So we + // have to wrap up |saSig|'s array of types in this API-matching class. + class MOZ_STACK_CLASS ItemsAndLength { + const MIRType* items_; + size_t length_; + + public: + ItemsAndLength(const MIRType* items, size_t length) + : items_(items), length_(length) {} + size_t length() const { return length_; } + MIRType operator[](size_t i) const { return items_[i]; } + }; + + // Assert, at least crudely, that we're not accidentally going to run off + // the end of the array of types, nor into undefined parts of it, while + // iterating. + MOZ_ASSERT(saSig.numArgs < + sizeof(saSig.argTypes) / sizeof(saSig.argTypes[0])); + MOZ_ASSERT(saSig.argTypes[saSig.numArgs] == MIRType::None /*the end marker*/); + + ItemsAndLength itemsAndLength(saSig.argTypes, saSig.numArgs); + return StackArgAreaSizeUnaligned(itemsAndLength); +} + +static inline size_t AlignStackArgAreaSize(size_t unalignedSize) { + return AlignBytes(unalignedSize, jit::WasmStackAlignment); +} + +// A stackmap creation helper. Create a stackmap from a vector of booleans. +// The caller owns the resulting stackmap. + +using StackMapBoolVector = Vector; + +wasm::StackMap* ConvertStackMapBoolVectorToStackMap( + const StackMapBoolVector& vec, bool hasRefs); + +// Generate a stackmap for a function's stack-overflow-at-entry trap, with +// the structure: +// +// +// | ++ +// | ++ +// | ++ +// | | +// Lowest Addr Highest Addr +// +// The caller owns the resulting stackmap. This assumes a grow-down stack. +// +// For non-debug builds, if the stackmap would contain no pointers, no +// stackmap is created, and nullptr is returned. For a debug build, a +// stackmap is always created and returned. +// +// The "space reserved before trap" is the space reserved by +// MacroAssembler::wasmReserveStackChecked, in the case where the frame is +// "small", as determined by that function. +[[nodiscard]] bool CreateStackMapForFunctionEntryTrap( + const ArgTypeVector& argTypes, const jit::RegisterOffsets& trapExitLayout, + size_t trapExitLayoutWords, size_t nBytesReservedBeforeTrap, + size_t nInboundStackArgBytes, wasm::StackMap** result); + +// At a resumable wasm trap, the machine's registers are saved on the stack by +// (code generated by) GenerateTrapExit(). This function writes into |args| a +// vector of booleans describing the ref-ness of the saved integer registers. +// |args[0]| corresponds to the low addressed end of the described section of +// the save area. +[[nodiscard]] bool GenerateStackmapEntriesForTrapExit( + const ArgTypeVector& args, const jit::RegisterOffsets& trapExitLayout, + const size_t trapExitLayoutNumWords, ExitStubMapVector* extras); + +// Shared write barrier code. +// +// A barriered store looks like this: +// +// Label skipPreBarrier; +// EmitWasmPreBarrierGuard(..., &skipPreBarrier); +// +// EmitWasmPreBarrierCall(...); +// bind(&skipPreBarrier); +// +// +// +// Label skipPostBarrier; +// +// EmitWasmPostBarrierGuard(..., &skipPostBarrier); +// +// bind(&skipPostBarrier); +// +// The actions are divided up to allow other actions to be placed between them, +// such as saving and restoring live registers. The postbarrier call invokes +// C++ and will kill all live registers. + +// Before storing a GC pointer value in memory, skip to `skipBarrier` if the +// prebarrier is not needed. Will clobber `scratch`. +// +// It is OK for `instance` and `scratch` to be the same register. +// +// If `trapOffset` is non-null, then metadata to catch a null access and emit +// a null pointer exception will be emitted. This will only catch a null access +// due to an incremental GC being in progress, the write that follows this +// pre-barrier guard must also be guarded against null. + +void EmitWasmPreBarrierGuard(jit::MacroAssembler& masm, Register instance, + Register scratch, Register valueAddr, + size_t valueOffset, Label* skipBarrier, + BytecodeOffset* trapOffset); + +// Before storing a GC pointer value in memory, call out-of-line prebarrier +// code. This assumes `PreBarrierReg` contains the address that will be updated. +// On ARM64 it also assums that x28 (the PseudoStackPointer) has the same value +// as SP. `PreBarrierReg` is preserved by the barrier function. Will clobber +// `scratch`. +// +// It is OK for `instance` and `scratch` to be the same register. + +void EmitWasmPreBarrierCall(jit::MacroAssembler& masm, Register instance, + Register scratch, Register valueAddr, + size_t valueOffset); + +// After storing a GC pointer value in memory, skip to `skipBarrier` if a +// postbarrier is not needed. If the location being set is in an heap-allocated +// object then `object` must reference that object; otherwise it should be None. +// The value that was stored is `setValue`. Will clobber `otherScratch` and +// will use other available scratch registers. +// +// `otherScratch` cannot be a designated scratch register. + +void EmitWasmPostBarrierGuard(jit::MacroAssembler& masm, + const mozilla::Maybe& object, + Register otherScratch, Register setValue, + Label* skipBarrier); + +#ifdef DEBUG +// Check whether |nextPC| is a valid code address for a stackmap created by +// this compiler. +bool IsValidStackMapKey(bool debugEnabled, const uint8_t* nextPC); +#endif + +} // namespace wasm +} // namespace js + +#endif // wasm_gc_h diff --git a/js/src/wasm/WasmGcObject.cpp b/js/src/wasm/WasmGcObject.cpp new file mode 100644 index 0000000000..6b2e5d40cf --- /dev/null +++ b/js/src/wasm/WasmGcObject.cpp @@ -0,0 +1,872 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "wasm/WasmGcObject.h" + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Casting.h" +#include "mozilla/CheckedInt.h" + +#include + +#include "gc/Marking.h" +#include "js/CharacterEncoding.h" +#include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_* +#include "js/PropertySpec.h" +#include "js/ScalarType.h" // js::Scalar::Type +#include "js/Vector.h" +#include "util/StringBuffer.h" +#include "vm/GlobalObject.h" +#include "vm/JSFunction.h" +#include "vm/JSObject.h" +#include "vm/PlainObject.h" // js::PlainObject +#include "vm/Realm.h" +#include "vm/SelfHosting.h" +#include "vm/StringType.h" +#include "vm/TypedArrayObject.h" +#include "vm/Uint8Clamped.h" + +#include "gc/GCContext-inl.h" +#include "gc/Marking-inl.h" +#include "gc/Nursery-inl.h" +#include "gc/StoreBuffer-inl.h" +#include "vm/JSAtom-inl.h" +#include "vm/JSObject-inl.h" +#include "vm/NativeObject-inl.h" +#include "vm/Shape-inl.h" + +using mozilla::AssertedCast; +using mozilla::CheckedUint32; +using mozilla::IsPowerOfTwo; +using mozilla::PodCopy; +using mozilla::PointerRangeSize; + +using namespace js; +using namespace wasm; + +// [SMDOC] Management of OOL storage areas for Wasm{Array,Struct}Object. +// +// WasmArrayObject always has its payload data stored in a block the C++-heap, +// which is pointed to from the WasmArrayObject. The same is true for +// WasmStructObject in the case where the fields cannot fit in the object +// itself. These C++ blocks are in some places referred to as "trailer blocks". +// +// The presence of trailer blocks complicates the use of generational GC (that +// is, Nursery allocation) of Wasm{Array,Struct}Object. In particular: +// +// (1) For objects which do not get tenured at minor collection, there must be +// a way to free the associated trailer, but there is no way to visit +// non-tenured blocks during minor collection. +// +// (2) Even if (1) were solved, calling js_malloc/js_free for every object +// creation-death cycle is expensive, possibly around 400 machine +// instructions, and we expressly want to avoid that in a generational GC +// scenario. +// +// The following scheme is therefore employed. +// +// (a) gc::Nursery maintains a pool of available C++-heap-allocated blocks -- +// a js::MallocedBlockCache -- and the intention is that trailers are +// allocated from this pool and freed back into it whenever possible. +// +// (b) WasmArrayObject::createArray and WasmStructObject::createStructOOL +// always request trailer allocation from the nursery's cache (a). If the +// cache cannot honour the request directly it will allocate directly from +// js_malloc; we hope this happens only infrequently. +// +// (c) The allocated block is returned as a js::PointerAndUint7, a pair that +// holds the trailer block pointer and an auxiliary tag that the +// js::MallocedBlockCache needs to see when the block is freed. +// +// The raw trailer block pointer (a `void*`) is stored in the +// Wasm{Array,Struct}Object OOL data field. These objects are not aware +// of and do not interact with js::PointerAndUint7, and nor does any +// JIT-generated code. +// +// (d) Still in WasmArrayObject::createArray and +// WasmStructObject::createStructOOL, if the object was allocated in the +// nursery, then the resulting js::PointerAndUint7 is "registered" with +// the nursery by handing it to Nursery::registerTrailer. +// +// (e) When a minor collection happens (Nursery::doCollection), we are +// notified of objects that are moved by calls to the ::obj_moved methods +// in this file. For those objects that have been tenured, the raw +// trailer pointer is "deregistered" with the nursery by handing it to +// Nursery::deregisterTrailer. +// +// (f) Still during minor collection: The nursery now knows both the set of +// trailer blocks added, and those removed because the corresponding +// object has been tenured. The difference between these two sets (that +// is, `added - removed`) is the set of trailer blocks corresponding to +// blocks that didn't get tenured. That set is computed and freed (back +// to the nursery's js::MallocedBlockCache) by +// :Nursery::freeTrailerBlocks. +// +// (g) At the end of minor collection, the added and removed sets are made +// empty, and the cycle begins again. +// +// (h) Also at the end of minor collection, a call to +// `mallocedBlockCache_.preen` hands a few blocks in the cache back to +// js_free. This mechanism exists so as to ensure that unused blocks do +// not remain in the cache indefinitely. +// +// (i) For objects that got tenured, we are eventually notified of their death +// by a call to the ::obj_finalize methods below. At that point we hand +// their block pointers to js_free. +// +// (j) When the nursery is eventually destroyed, all blocks in its block cache +// are handed to js_free. Hence, at process exit, provided all nurseries +// are first collected and then their destructors run, no C++ heap blocks +// are leaked. +// +// As a result of this scheme, trailer blocks associated with what we hope is +// the frequent case -- objects that are allocated but never make it out of +// the nursery -- are cycled through the nursery's block cache. +// +// Trailers associated with tenured blocks cannot participate though; they are +// always returned to js_free. It would be possible to enable them to +// participate by changing their owning object's OOL data pointer to be a +// js::PointerAndUint7 rather than a raw `void*`, so that then the blocks +// could be released to the cache in the ::obj_finalize methods. This would +// however require changes in the generated code for array element and OOL +// struct element accesses. +// +// Here's a short summary of the trailer block life cycle: +// +// * allocated: +// +// - in WasmArrayObject::createArray / WasmStructObject::createStructOOL +// +// - by calling the nursery's MallocBlockCache alloc method +// +// * deallocated: +// +// - for non-tenured objects, in the collector itself, +// in Nursery::doCollection calling Nursery::freeTrailerBlocks, +// releasing to the nursery's block cache +// +// - for tenured objects, in the ::obj_finalize methods, releasing directly +// to js_free +// +// If this seems confusing ("why is it ok to allocate from the cache but +// release to js_free?"), remember that the cache holds blocks previously +// obtained from js_malloc but which are *not* currently in use. Hence it is +// fine to give them back to js_free; that just makes the cache a bit emptier +// but has no effect on correctness. + +//========================================================================= +// WasmGcObject + +bool WasmGcObject::lookupProperty(JSContext* cx, Handle object, + jsid id, PropOffset* offset, + FieldType* type) { + switch (kind()) { + case wasm::TypeDefKind::Struct: { + const auto& structType = typeDef().structType(); + uint32_t index; + if (!IdIsIndex(id, &index)) { + return false; + } + if (index >= structType.fields_.length()) { + return false; + } + const StructField& field = structType.fields_[index]; + offset->set(field.offset); + *type = field.type; + return true; + } + case wasm::TypeDefKind::Array: { + const auto& arrayType = typeDef().arrayType(); + + // Special case for property 'length' that loads the length field at the + // beginning of the data buffer + if (id.isString() && + id.toString() == cx->runtime()->commonNames->length) { + STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; + *type = FieldType::I32; + offset->set(UINT32_MAX); + return true; + } + + // Normal case of indexed properties for loading array elements + uint32_t index; + if (!IdIsIndex(id, &index)) { + return false; + } + uint32_t numElements = object->as().numElements_; + if (index >= numElements) { + return false; + } + uint64_t scaledIndex = + uint64_t(index) * uint64_t(arrayType.elementType_.size()); + if (scaledIndex >= uint64_t(UINT32_MAX)) { + // It's unrepresentable as an WasmGcObject::PropOffset. Give up. + return false; + } + offset->set(uint32_t(scaledIndex)); + *type = arrayType.elementType_; + return true; + } + default: + MOZ_ASSERT_UNREACHABLE(); + return false; + } +} + +const ObjectOps WasmGcObject::objectOps_ = { + WasmGcObject::obj_lookupProperty, // lookupProperty + WasmGcObject::obj_defineProperty, // defineProperty + WasmGcObject::obj_hasProperty, // hasProperty + WasmGcObject::obj_getProperty, // getProperty + WasmGcObject::obj_setProperty, // setProperty + WasmGcObject::obj_getOwnPropertyDescriptor, // getOwnPropertyDescriptor + WasmGcObject::obj_deleteProperty, // deleteProperty + nullptr, // getElements + nullptr, // funToString +}; + +/* static */ +bool WasmGcObject::obj_lookupProperty(JSContext* cx, HandleObject obj, + HandleId id, MutableHandleObject objp, + PropertyResult* propp) { + Rooted typedObj(cx, &obj->as()); + if (typedObj->hasProperty(cx, typedObj, id)) { + propp->setWasmGcProperty(); + objp.set(obj); + return true; + } + + RootedObject proto(cx, obj->staticPrototype()); + if (!proto) { + objp.set(nullptr); + propp->setNotFound(); + return true; + } + + return LookupProperty(cx, proto, id, objp, propp); +} + +bool WasmGcObject::obj_defineProperty(JSContext* cx, HandleObject obj, + HandleId id, + Handle desc, + ObjectOpResult& result) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_OBJECT_NOT_EXTENSIBLE, "WasmGcObject"); + return false; +} + +bool WasmGcObject::obj_hasProperty(JSContext* cx, HandleObject obj, HandleId id, + bool* foundp) { + Rooted typedObj(cx, &obj->as()); + if (typedObj->hasProperty(cx, typedObj, id)) { + *foundp = true; + return true; + } + + RootedObject proto(cx, obj->staticPrototype()); + if (!proto) { + *foundp = false; + return true; + } + + return HasProperty(cx, proto, id, foundp); +} + +bool WasmGcObject::obj_getProperty(JSContext* cx, HandleObject obj, + HandleValue receiver, HandleId id, + MutableHandleValue vp) { + Rooted typedObj(cx, &obj->as()); + + WasmGcObject::PropOffset offset; + FieldType type; + if (typedObj->lookupProperty(cx, typedObj, id, &offset, &type)) { + return typedObj->loadValue(cx, offset, type, vp); + } + + RootedObject proto(cx, obj->staticPrototype()); + if (!proto) { + vp.setUndefined(); + return true; + } + + return GetProperty(cx, proto, receiver, id, vp); +} + +bool WasmGcObject::obj_setProperty(JSContext* cx, HandleObject obj, HandleId id, + HandleValue v, HandleValue receiver, + ObjectOpResult& result) { + Rooted typedObj(cx, &obj->as()); + + if (typedObj->hasProperty(cx, typedObj, id)) { + if (!receiver.isObject() || obj != &receiver.toObject()) { + return SetPropertyByDefining(cx, id, v, receiver, result); + } + + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_TYPEDOBJECT_SETTING_IMMUTABLE); + return false; + } + + return SetPropertyOnProto(cx, obj, id, v, receiver, result); +} + +bool WasmGcObject::obj_getOwnPropertyDescriptor( + JSContext* cx, HandleObject obj, HandleId id, + MutableHandle> desc) { + Rooted typedObj(cx, &obj->as()); + + WasmGcObject::PropOffset offset; + FieldType type; + if (typedObj->lookupProperty(cx, typedObj, id, &offset, &type)) { + RootedValue value(cx); + if (!typedObj->loadValue(cx, offset, type, &value)) { + return false; + } + desc.set(mozilla::Some(PropertyDescriptor::Data( + value, + {JS::PropertyAttribute::Enumerable, JS::PropertyAttribute::Writable}))); + return true; + } + + desc.reset(); + return true; +} + +bool WasmGcObject::obj_deleteProperty(JSContext* cx, HandleObject obj, + HandleId id, ObjectOpResult& result) { + Rooted typedObj(cx, &obj->as()); + if (typedObj->hasProperty(cx, typedObj, id)) { + return Throw(cx, id, JSMSG_CANT_DELETE); + } + + RootedObject proto(cx, obj->staticPrototype()); + if (!proto) { + return result.succeed(); + } + + return DeleteProperty(cx, proto, id, result); +} + +/* static */ +WasmGcObject* WasmGcObject::create(JSContext* cx, + wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap) { + MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp)); + MOZ_ASSERT(!typeDefData->clasp->isNativeObject()); + + debugCheckNewObject(typeDefData->shape, typeDefData->allocKind, initialHeap); + + WasmGcObject* obj = + cx->newCell(typeDefData->allocKind, initialHeap, + typeDefData->clasp, &typeDefData->allocSite); + if (!obj) { + return nullptr; + } + + obj->initShape(typeDefData->shape); + obj->superTypeVector_ = typeDefData->superTypeVector; + + js::gc::gcprobes::CreateObject(obj); + probes::CreateObject(cx, obj); + + return obj; +} + +bool WasmGcObject::loadValue(JSContext* cx, + const WasmGcObject::PropOffset& offset, + FieldType type, MutableHandleValue vp) { + // Temporary hack, (ref T) is not exposable to JS yet but some tests would + // like to access it so we erase (ref T) with eqref when loading. This is + // safe as (ref T) <: eqref and we're not in the writing case where we + // would need to perform a type check. + if (type.isTypeRef()) { + type = RefType::fromTypeCode(TypeCode::EqRef, true); + } + + if (!type.isExposable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + + if (is()) { + // `offset` is the field offset, without regard to the in/out-line split. + // That is handled by the call to `fieldOffsetToAddress`. + WasmStructObject& structObj = as(); + // Ensure no out-of-range access possible + MOZ_RELEASE_ASSERT(structObj.kind() == TypeDefKind::Struct); + MOZ_RELEASE_ASSERT(offset.get() + type.size() <= + structObj.typeDef().structType().size_); + return ToJSValue(cx, structObj.fieldOffsetToAddress(type, offset.get()), + type, vp); + } + + MOZ_ASSERT(is()); + WasmArrayObject& arrayObj = as(); + if (offset.get() == UINT32_MAX) { + // This denotes "length" + uint32_t numElements = arrayObj.numElements_; + // We can't use `ToJSValue(.., ValType::I32, ..)` here since it will + // treat the integer as signed, which it isn't. `vp.set(..)` will + // coerce correctly to a JS::Value, though. + vp.set(NumberValue(numElements)); + return true; + } + return ToJSValue(cx, arrayObj.data_ + offset.get(), type, vp); +} + +bool WasmGcObject::isRuntimeSubtypeOf( + const wasm::TypeDef* parentTypeDef) const { + return TypeDef::isSubTypeOf(&typeDef(), parentTypeDef); +} + +bool WasmGcObject::obj_newEnumerate(JSContext* cx, HandleObject obj, + MutableHandleIdVector properties, + bool enumerableOnly) { + MOZ_ASSERT(obj->is()); + Rooted typedObj(cx, &obj->as()); + + size_t indexCount = 0; + size_t otherCount = 0; + switch (typedObj->kind()) { + case wasm::TypeDefKind::Struct: { + indexCount = typedObj->typeDef().structType().fields_.length(); + break; + } + case wasm::TypeDefKind::Array: { + indexCount = typedObj->as().numElements_; + otherCount = 1; + break; + } + default: + MOZ_ASSERT_UNREACHABLE(); + } + + if (!properties.reserve(indexCount + otherCount)) { + return false; + } + RootedId id(cx); + for (size_t index = 0; index < indexCount; index++) { + id = PropertyKey::Int(int32_t(index)); + properties.infallibleAppend(id); + } + + if (typedObj->kind() == wasm::TypeDefKind::Array) { + properties.infallibleAppend(NameToId(cx->runtime()->commonNames->length)); + } + + return true; +} + +static void WriteValTo(const Val& val, FieldType ty, void* dest) { + switch (ty.kind()) { + case FieldType::I8: + *((uint8_t*)dest) = val.i32(); + break; + case FieldType::I16: + *((uint16_t*)dest) = val.i32(); + break; + case FieldType::I32: + *((uint32_t*)dest) = val.i32(); + break; + case FieldType::I64: + *((uint64_t*)dest) = val.i64(); + break; + case FieldType::F32: + *((float*)dest) = val.f32(); + break; + case FieldType::F64: + *((double*)dest) = val.f64(); + break; + case FieldType::V128: + *((V128*)dest) = val.v128(); + break; + case FieldType::Ref: + *((GCPtr*)dest) = val.ref(); + break; + } +} + +//========================================================================= +// WasmArrayObject + +/* static */ +gc::AllocKind WasmArrayObject::allocKind() { + return gc::GetGCObjectKindForBytes(sizeof(WasmArrayObject)); +} + +/* static */ +template +WasmArrayObject* WasmArrayObject::createArray( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap, uint32_t numElements) { + const TypeDef* typeDef = typeDefData->typeDef; + STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; + MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Array); + + // Calculate the byte length of the outline storage, being careful to check + // for overflow. Note this logic assumes that MaxArrayPayloadBytes is + // within uint32_t range. + CheckedUint32 outlineBytes = typeDef->arrayType().elementType_.size(); + outlineBytes *= numElements; + if (!outlineBytes.isValid() || + outlineBytes.value() > uint32_t(MaxArrayPayloadBytes)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_ARRAY_IMP_LIMIT); + return nullptr; + } + + // Allocate the outline data before allocating the object so that we can + // infallibly initialize the pointer on the array object after it is + // allocated. + Nursery& nursery = cx->nursery(); + PointerAndUint7 outlineData(nullptr, 0); + if (outlineBytes.value() > 0) { + outlineData = nursery.mallocedBlockCache().alloc(outlineBytes.value()); + if (!outlineData.pointer()) { + ReportOutOfMemory(cx); + return nullptr; + } + } + + // It's unfortunate that `arrayObj` has to be rooted, since this is a hot + // path and rooting costs around 15 instructions. It is the call to + // registerTrailer that makes it necessary. + Rooted arrayObj(cx); + arrayObj = + (WasmArrayObject*)WasmGcObject::create(cx, typeDefData, initialHeap); + if (!arrayObj) { + ReportOutOfMemory(cx); + if (outlineData.pointer()) { + nursery.mallocedBlockCache().free(outlineData); + } + return nullptr; + } + + arrayObj->numElements_ = numElements; + arrayObj->data_ = (uint8_t*)outlineData.pointer(); + if (outlineData.pointer()) { + if constexpr (ZeroFields) { + memset(outlineData.pointer(), 0, outlineBytes.value()); + } + if (js::gc::IsInsideNursery(arrayObj)) { + // We need to register the OOL area with the nursery, so it will be + // freed after GCing of the nursery if `arrayObj_` doesn't make it into + // the tenured heap. + if (!nursery.registerTrailer(outlineData, outlineBytes.value())) { + nursery.mallocedBlockCache().free(outlineData); + ReportOutOfMemory(cx); + return nullptr; + } + } + } + + return arrayObj; +} + +template WasmArrayObject* WasmArrayObject::createArray( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap, uint32_t numElements); +template WasmArrayObject* WasmArrayObject::createArray( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap, uint32_t numElements); + +/* static */ +void WasmArrayObject::obj_trace(JSTracer* trc, JSObject* object) { + WasmArrayObject& arrayObj = object->as(); + uint8_t* data = arrayObj.data_; + if (!data) { + MOZ_ASSERT(arrayObj.numElements_ == 0); + return; + } + + const auto& typeDef = arrayObj.typeDef(); + const auto& arrayType = typeDef.arrayType(); + if (!arrayType.elementType_.isRefRepr()) { + return; + } + + uint32_t numElements = arrayObj.numElements_; + MOZ_ASSERT(numElements > 0); + uint32_t elemSize = arrayType.elementType_.size(); + for (uint32_t i = 0; i < numElements; i++) { + GCPtr* objectPtr = + reinterpret_cast*>(data + i * elemSize); + TraceNullableEdge(trc, objectPtr, "reference-obj"); + } +} + +/* static */ +void WasmArrayObject::obj_finalize(JS::GCContext* gcx, JSObject* object) { + WasmArrayObject& arrayObj = object->as(); + MOZ_ASSERT((arrayObj.data_ == nullptr) == (arrayObj.numElements_ == 0)); + if (arrayObj.data_) { + js_free(arrayObj.data_); + arrayObj.data_ = nullptr; + } +} + +/* static */ +size_t WasmArrayObject::obj_moved(JSObject* obj, JSObject* old) { + MOZ_ASSERT(!IsInsideNursery(obj)); + if (IsInsideNursery(old)) { + // It's been tenured. + MOZ_ASSERT(obj->isTenured()); + WasmArrayObject& arrayObj = obj->as(); + if (arrayObj.data_) { + Nursery& nursery = obj->runtimeFromMainThread()->gc.nursery(); + nursery.unregisterTrailer(arrayObj.data_); + } + } + return 0; +} + +void WasmArrayObject::storeVal(const Val& val, uint32_t itemIndex) { + const ArrayType& arrayType = typeDef().arrayType(); + size_t elementSize = arrayType.elementType_.size(); + MOZ_ASSERT(itemIndex < numElements_); + uint8_t* data = data_ + elementSize * itemIndex; + WriteValTo(val, arrayType.elementType_, data); +} + +void WasmArrayObject::fillVal(const Val& val, uint32_t itemIndex, + uint32_t len) { + const ArrayType& arrayType = typeDef().arrayType(); + size_t elementSize = arrayType.elementType_.size(); + uint8_t* data = data_ + elementSize * itemIndex; + MOZ_ASSERT(itemIndex <= numElements_ && len <= numElements_ - itemIndex); + for (uint32_t i = 0; i < len; i++) { + WriteValTo(val, arrayType.elementType_, data); + data += elementSize; + } +} + +static const JSClassOps WasmArrayObjectClassOps = { + nullptr, /* addProperty */ + nullptr, /* delProperty */ + nullptr, /* enumerate */ + WasmGcObject::obj_newEnumerate, + nullptr, /* resolve */ + nullptr, /* mayResolve */ + WasmArrayObject::obj_finalize, /* finalize */ + nullptr, /* call */ + nullptr, /* construct */ + WasmArrayObject::obj_trace, +}; +static const ClassExtension WasmArrayObjectClassExt = { + WasmArrayObject::obj_moved /* objectMovedOp */ +}; +const JSClass WasmArrayObject::class_ = { + "WasmArrayObject", + JSClass::NON_NATIVE | JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_BACKGROUND_FINALIZE | JSCLASS_SKIP_NURSERY_FINALIZE, + &WasmArrayObjectClassOps, + JS_NULL_CLASS_SPEC, + &WasmArrayObjectClassExt, + &WasmGcObject::objectOps_}; + +//========================================================================= +// WasmStructObject + +/* static */ +const JSClass* js::WasmStructObject::classForTypeDef( + const wasm::TypeDef* typeDef) { + MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct); + size_t nbytes = typeDef->structType().size_; + return nbytes > WasmStructObject_MaxInlineBytes + ? &WasmStructObject::classOutline_ + : &WasmStructObject::classInline_; +} + +/* static */ +js::gc::AllocKind js::WasmStructObject::allocKindForTypeDef( + const wasm::TypeDef* typeDef) { + MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct); + size_t nbytes = typeDef->structType().size_; + + // `nbytes` is the total required size for all struct fields, including + // padding. What we need is the size of resulting WasmStructObject, + // ignoring any space used for out-of-line data. First, restrict `nbytes` + // to cover just the inline data. + if (nbytes > WasmStructObject_MaxInlineBytes) { + nbytes = WasmStructObject_MaxInlineBytes; + } + + // Now convert it to size of the WasmStructObject as a whole. + nbytes = sizeOfIncludingInlineData(nbytes); + + return gc::GetGCObjectKindForBytes(nbytes); +} + +/* static MOZ_NEVER_INLINE */ +template +WasmStructObject* WasmStructObject::createStructOOL( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap, uint32_t inlineBytes, uint32_t outlineBytes) { + // This method is called as the slow path from the (inlineable) + // WasmStructObject::createStruct. It handles the case where an object + // needs OOL storage. It doesn't handle the non-OOL case at all. + + // Allocate the outline data area before allocating the object so that we can + // infallibly initialize the outline data area. + Nursery& nursery = cx->nursery(); + PointerAndUint7 outlineData = + nursery.mallocedBlockCache().alloc(outlineBytes); + if (MOZ_UNLIKELY(!outlineData.pointer())) { + ReportOutOfMemory(cx); + return nullptr; + } + + // See corresponding comment in WasmArrayObject::createArray. + Rooted structObj(cx); + structObj = + (WasmStructObject*)WasmGcObject::create(cx, typeDefData, initialHeap); + if (MOZ_UNLIKELY(!structObj)) { + ReportOutOfMemory(cx); + if (outlineData.pointer()) { + nursery.mallocedBlockCache().free(outlineData); + } + return nullptr; + } + + // Initialize the outline data fields + structObj->outlineData_ = (uint8_t*)outlineData.pointer(); + if constexpr (ZeroFields) { + memset(&(structObj->inlineData_[0]), 0, inlineBytes); + memset(outlineData.pointer(), 0, outlineBytes); + } + if (MOZ_LIKELY(js::gc::IsInsideNursery(structObj))) { + // See corresponding comment in WasmArrayObject::createArray. + if (!nursery.registerTrailer(outlineData, outlineBytes)) { + nursery.mallocedBlockCache().free(outlineData); + ReportOutOfMemory(cx); + return nullptr; + } + } + + return structObj; +} + +template WasmStructObject* WasmStructObject::createStruct( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap); +template WasmStructObject* WasmStructObject::createStruct( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap); + +/* static */ +void WasmStructObject::obj_trace(JSTracer* trc, JSObject* object) { + WasmStructObject& structObj = object->as(); + + const auto& structType = structObj.typeDef().structType(); + for (uint32_t offset : structType.inlineTraceOffsets_) { + GCPtr* objectPtr = + reinterpret_cast*>(&structObj.inlineData_[0] + offset); + TraceNullableEdge(trc, objectPtr, "reference-obj"); + } + for (uint32_t offset : structType.outlineTraceOffsets_) { + GCPtr* objectPtr = + reinterpret_cast*>(structObj.outlineData_ + offset); + TraceNullableEdge(trc, objectPtr, "reference-obj"); + } +} + +/* static */ +void WasmStructObject::obj_finalize(JS::GCContext* gcx, JSObject* object) { + WasmStructObject& structObj = object->as(); + + if (structObj.outlineData_) { + js_free(structObj.outlineData_); + structObj.outlineData_ = nullptr; + } +} + +/* static */ +size_t WasmStructObject::obj_moved(JSObject* obj, JSObject* old) { + MOZ_ASSERT(!IsInsideNursery(obj)); + if (IsInsideNursery(old)) { + // It's been tenured. + MOZ_ASSERT(obj->isTenured()); + WasmStructObject& structObj = obj->as(); + // WasmStructObject::classForTypeDef ensures we only get called for + // structs with OOL data. Hence: + MOZ_ASSERT(structObj.outlineData_); + Nursery& nursery = obj->runtimeFromMainThread()->gc.nursery(); + nursery.unregisterTrailer(structObj.outlineData_); + } + return 0; +} + +void WasmStructObject::storeVal(const Val& val, uint32_t fieldIndex) { + const StructType& structType = typeDef().structType(); + FieldType fieldType = structType.fields_[fieldIndex].type; + uint32_t fieldOffset = structType.fields_[fieldIndex].offset; + + MOZ_ASSERT(fieldIndex < structType.fields_.length()); + bool areaIsOutline; + uint32_t areaOffset; + fieldOffsetToAreaAndOffset(fieldType, fieldOffset, &areaIsOutline, + &areaOffset); + + uint8_t* data; + if (areaIsOutline) { + data = outlineData_ + areaOffset; + } else { + data = inlineData_ + areaOffset; + } + + WriteValTo(val, fieldType, data); +} + +static const JSClassOps WasmStructObjectOutlineClassOps = { + nullptr, /* addProperty */ + nullptr, /* delProperty */ + nullptr, /* enumerate */ + WasmGcObject::obj_newEnumerate, + nullptr, /* resolve */ + nullptr, /* mayResolve */ + WasmStructObject::obj_finalize, /* finalize */ + nullptr, /* call */ + nullptr, /* construct */ + WasmStructObject::obj_trace, +}; +static const ClassExtension WasmStructObjectOutlineClassExt = { + WasmStructObject::obj_moved /* objectMovedOp */ +}; +const JSClass WasmStructObject::classOutline_ = { + "WasmStructObject", + JSClass::NON_NATIVE | JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_BACKGROUND_FINALIZE | JSCLASS_SKIP_NURSERY_FINALIZE, + &WasmStructObjectOutlineClassOps, + JS_NULL_CLASS_SPEC, + &WasmStructObjectOutlineClassExt, + &WasmGcObject::objectOps_}; + +// Structs that only have inline data get a different class without a +// finalizer. This class should otherwise be identical to the class for +// structs with outline data. +static const JSClassOps WasmStructObjectInlineClassOps = { + nullptr, /* addProperty */ + nullptr, /* delProperty */ + nullptr, /* enumerate */ + WasmGcObject::obj_newEnumerate, + nullptr, /* resolve */ + nullptr, /* mayResolve */ + nullptr, /* finalize */ + nullptr, /* call */ + nullptr, /* construct */ + WasmStructObject::obj_trace, +}; +static const ClassExtension WasmStructObjectInlineClassExt = { + nullptr /* objectMovedOp */ +}; +const JSClass WasmStructObject::classInline_ = { + "WasmStructObject", + JSClass::NON_NATIVE | JSCLASS_DELAY_METADATA_BUILDER, + &WasmStructObjectInlineClassOps, + JS_NULL_CLASS_SPEC, + &WasmStructObjectInlineClassExt, + &WasmGcObject::objectOps_}; diff --git a/js/src/wasm/WasmGcObject.h b/js/src/wasm/WasmGcObject.h new file mode 100644 index 0000000000..f3d82c97fe --- /dev/null +++ b/js/src/wasm/WasmGcObject.h @@ -0,0 +1,396 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef wasm_WasmGcObject_h +#define wasm_WasmGcObject_h + +#include "mozilla/CheckedInt.h" +#include "mozilla/Maybe.h" + +#include "gc/Allocator.h" +#include "gc/Pretenuring.h" +#include "vm/ArrayBufferObject.h" +#include "vm/JSObject.h" +#include "wasm/WasmInstanceData.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValType.h" + +using js::wasm::FieldType; + +namespace js { + +//========================================================================= +// WasmGcObject + +class WasmGcObject : public JSObject { + protected: + const wasm::SuperTypeVector* superTypeVector_; + + static const ObjectOps objectOps_; + + [[nodiscard]] static bool obj_lookupProperty(JSContext* cx, HandleObject obj, + HandleId id, + MutableHandleObject objp, + PropertyResult* propp); + + [[nodiscard]] static bool obj_defineProperty(JSContext* cx, HandleObject obj, + HandleId id, + Handle desc, + ObjectOpResult& result); + + [[nodiscard]] static bool obj_hasProperty(JSContext* cx, HandleObject obj, + HandleId id, bool* foundp); + + [[nodiscard]] static bool obj_getProperty(JSContext* cx, HandleObject obj, + HandleValue receiver, HandleId id, + MutableHandleValue vp); + + [[nodiscard]] static bool obj_setProperty(JSContext* cx, HandleObject obj, + HandleId id, HandleValue v, + HandleValue receiver, + ObjectOpResult& result); + + [[nodiscard]] static bool obj_getOwnPropertyDescriptor( + JSContext* cx, HandleObject obj, HandleId id, + MutableHandle> desc); + + [[nodiscard]] static bool obj_deleteProperty(JSContext* cx, HandleObject obj, + HandleId id, + ObjectOpResult& result); + + // PropOffset is a uint32_t that is used to carry information about the + // location of an value from WasmGcObject::lookupProperty to + // WasmGcObject::loadValue. It is distinct from a normal uint32_t to + // emphasise the fact that it cannot be interpreted as an offset in any + // single contiguous area of memory: + // + // * If the object in question is a WasmStructObject, it is the value of + // `wasm::StructField::offset` for the relevant field, without regard to + // the inline/outline split. + // + // * If the object in question is a WasmArrayObject, then + // - u32 == UINT32_MAX (0xFFFF'FFFF) means the "length" property + // is requested + // - u32 < UINT32_MAX means the array element starting at that byte + // offset in WasmArrayObject::data_. It is not an array index value. + // See WasmGcObject::lookupProperty for details. + class PropOffset { + uint32_t u32_; + + public: + PropOffset() : u32_(0) {} + uint32_t get() const { return u32_; } + void set(uint32_t u32) { u32_ = u32; } + }; + + [[nodiscard]] bool lookupProperty(JSContext* cx, + js::Handle object, jsid id, + PropOffset* offset, wasm::FieldType* type); + [[nodiscard]] bool hasProperty(JSContext* cx, + js::Handle object, jsid id) { + WasmGcObject::PropOffset offset; + wasm::FieldType type; + return lookupProperty(cx, object, id, &offset, &type); + } + + bool loadValue(JSContext* cx, const WasmGcObject::PropOffset& offset, + wasm::FieldType type, MutableHandleValue vp); + + public: + const wasm::SuperTypeVector& superTypeVector() const { + return *superTypeVector_; + } + + static size_t offsetOfSuperTypeVector() { + return offsetof(WasmGcObject, superTypeVector_); + } + + // These are both expensive in that they involve a double indirection. + // Avoid them if possible. + const wasm::TypeDef& typeDef() const { return *superTypeVector().typeDef(); } + wasm::TypeDefKind kind() const { return superTypeVector().typeDef()->kind(); } + + [[nodiscard]] bool isRuntimeSubtypeOf( + const wasm::TypeDef* parentTypeDef) const; + + [[nodiscard]] static bool obj_newEnumerate(JSContext* cx, HandleObject obj, + MutableHandleIdVector properties, + bool enumerableOnly); + + protected: + // Create the GcObject (struct/array-specific fields are uninitialised). + // The type, shape, class pointer, alloc site and alloc kind are taken + // from `typeDefData`; the initial heap must be specified separately. + static WasmGcObject* create(JSContext* cx, + wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap); +}; + +//========================================================================= +// WasmArrayObject + +// Class for a wasm array. It contains a pointer to the array contents, that +// lives in the C++ heap. + +class WasmArrayObject : public WasmGcObject { + public: + static const JSClass class_; + + // The number of elements in the array. + uint32_t numElements_; + + // Owned data pointer, holding `numElements_` entries. This is null if + // `numElements_` is zero; otherwise it must be non-null. See bug 1812283. + uint8_t* data_; + + // AllocKind for object creation + static gc::AllocKind allocKind(); + + // Creates a new array typed object, optionally initialized to zero, for the + // specified number of elements. Reports an error if the number of elements + // is too large, or if there is an out of memory error. The element type, + // shape, class pointer, alloc site and alloc kind are taken from + // `typeDefData`; the initial heap must be specified separately. + template + static WasmArrayObject* createArray(JSContext* cx, + wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap, + uint32_t numElements); + + // JIT accessors + static constexpr size_t offsetOfNumElements() { + return offsetof(WasmArrayObject, numElements_); + } + static constexpr size_t offsetOfData() { + return offsetof(WasmArrayObject, data_); + } + + // Tracing and finalization + static void obj_trace(JSTracer* trc, JSObject* object); + static void obj_finalize(JS::GCContext* gcx, JSObject* object); + static size_t obj_moved(JSObject* obj, JSObject* old); + + void storeVal(const wasm::Val& val, uint32_t itemIndex); + void fillVal(const wasm::Val& val, uint32_t itemIndex, uint32_t len); +}; + +// Helper to mark all locations that assume that the type of +// WasmArrayObject::numElements is uint32_t. +#define STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32 \ + static_assert(sizeof(js::WasmArrayObject::numElements_) == sizeof(uint32_t)) + +//========================================================================= +// WasmStructObject + +// Class for a wasm struct. It has inline data and, if the inline area is +// insufficient, a pointer to outline data that lives in the C++ heap. +// Computing the field offsets is somewhat tricky; see block comment on `class +// StructLayout` for background. + +class WasmStructObject : public WasmGcObject { + public: + static const JSClass classInline_; + static const JSClass classOutline_; + + // Owned pointer to a malloc'd block containing out-of-line fields, or + // nullptr if none. Note that MIR alias analysis assumes this is readonly + // for the life of the object; do not change it once the object is created. + // See MWasmLoadObjectField::congruentTo. + uint8_t* outlineData_; + + // The inline (wasm-struct-level) data fields. This must be a multiple of + // 16 bytes long in order to ensure that no field gets split across the + // inline-outline boundary. As a refinement, we request this field to begin + // at an 8-aligned offset relative to the start of the object, so as to + // guarantee that `double` typed fields are not subject to misaligned-access + // penalties on any target, whilst wasting at maximum 4 bytes of space. + // + // `inlineData_` is in reality a variable length block with maximum size + // WasmStructObject_MaxInlineBytes bytes. Do not add any (C++-level) fields + // after this point! + alignas(8) uint8_t inlineData_[0]; + + // This tells us how big the object is if we know the number of inline bytes + // it was created with. + static inline size_t sizeOfIncludingInlineData(size_t sizeOfInlineData) { + size_t n = sizeof(WasmStructObject) + sizeOfInlineData; + MOZ_ASSERT(n <= JSObject::MAX_BYTE_SIZE); + return n; + } + + static const JSClass* classForTypeDef(const wasm::TypeDef* typeDef); + static js::gc::AllocKind allocKindForTypeDef(const wasm::TypeDef* typeDef); + + // Slow path for ::createStruct, in which we know we need to generate a + // struct with an out-of-line storage area. This is marked as never-inline + // so as to maximise the chance that WasmStructObject::createStruct itself + // can be inlined. + template + static MOZ_NEVER_INLINE WasmStructObject* createStructOOL( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap, uint32_t inlineBytes, uint32_t outlineBytes); + + // Creates a new struct typed object, optionally initialized to zero. + // Reports if there is an out of memory error. The structure's type, shape, + // class pointer, alloc site and alloc kind are taken from `typeDefData`; + // the initial heap must be specified separately. For structs with no OOL + // storage requirement, the required work is accomplished in-line; otherwise + // we slow-path to WasmStructObject::createStructOOL. + template + static inline WasmStructObject* createStruct( + JSContext* cx, wasm::TypeDefInstanceData* typeDefData, + js::gc::Heap initialHeap) { + const wasm::TypeDef* typeDef = typeDefData->typeDef; + MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct); + + uint32_t totalBytes = typeDef->structType().size_; + uint32_t inlineBytes, outlineBytes; + WasmStructObject::getDataByteSizes(totalBytes, &inlineBytes, &outlineBytes); + + if (MOZ_LIKELY(outlineBytes == 0)) { + // This doesn't need to be rooted, since all we do with it prior to + // return is to zero out the fields (and then only if ZeroFields is true). + WasmStructObject* structObj = + (WasmStructObject*)WasmGcObject::create(cx, typeDefData, initialHeap); + if (MOZ_UNLIKELY(!structObj)) { + ReportOutOfMemory(cx); + return nullptr; + } + + structObj->outlineData_ = nullptr; + if constexpr (ZeroFields) { + memset(&(structObj->inlineData_[0]), 0, inlineBytes); + } + return structObj; + } + + // OOL storage is required, so hand off to non-inlined code. + return WasmStructObject::createStructOOL( + cx, typeDefData, initialHeap, inlineBytes, outlineBytes); + } + + // Given the total number of data bytes required (including alignment + // holes), return the number of inline and outline bytes required. + static inline void getDataByteSizes(uint32_t totalBytes, + uint32_t* inlineBytes, + uint32_t* outlineBytes); + + // Given the offset of a field, produce the offset in `inlineData_` or + // `*outlineData_` to use, plus a bool indicating which area it is. + // `fieldType` is for assertional purposes only. + static inline void fieldOffsetToAreaAndOffset(FieldType fieldType, + uint32_t fieldOffset, + bool* areaIsOutline, + uint32_t* areaOffset); + + // Given the offset of a field, return its actual address. `fieldType` is + // for assertional purposes only. + inline uint8_t* fieldOffsetToAddress(FieldType fieldType, + uint32_t fieldOffset); + + // JIT accessors + static constexpr size_t offsetOfOutlineData() { + return offsetof(WasmStructObject, outlineData_); + } + static constexpr size_t offsetOfInlineData() { + return offsetof(WasmStructObject, inlineData_); + } + + // Tracing and finalization + static void obj_trace(JSTracer* trc, JSObject* object); + static void obj_finalize(JS::GCContext* gcx, JSObject* object); + static size_t obj_moved(JSObject* obj, JSObject* old); + + void storeVal(const wasm::Val& val, uint32_t fieldIndex); +}; + +// This is ensured by the use of `alignas` on `WasmStructObject::inlineData_`. +static_assert((offsetof(WasmStructObject, inlineData_) % 8) == 0); + +// MaxInlineBytes must be a multiple of 16 for reasons described in the +// comment on `class StructLayout`. This unfortunately can't be defined +// inside the class definition itself because the sizeof(..) expression isn't +// valid until after the end of the class definition. +const size_t WasmStructObject_MaxInlineBytes = + ((JSObject::MAX_BYTE_SIZE - sizeof(WasmStructObject)) / 16) * 16; + +static_assert((WasmStructObject_MaxInlineBytes % 16) == 0); + +/*static*/ +inline void WasmStructObject::getDataByteSizes(uint32_t totalBytes, + uint32_t* inlineBytes, + uint32_t* outlineBytes) { + if (MOZ_UNLIKELY(totalBytes > WasmStructObject_MaxInlineBytes)) { + *inlineBytes = WasmStructObject_MaxInlineBytes; + *outlineBytes = totalBytes - WasmStructObject_MaxInlineBytes; + } else { + *inlineBytes = totalBytes; + *outlineBytes = 0; + } +} + +/*static*/ +inline void WasmStructObject::fieldOffsetToAreaAndOffset(FieldType fieldType, + uint32_t fieldOffset, + bool* areaIsOutline, + uint32_t* areaOffset) { + if (fieldOffset < WasmStructObject_MaxInlineBytes) { + *areaIsOutline = false; + *areaOffset = fieldOffset; + } else { + *areaIsOutline = true; + *areaOffset = fieldOffset - WasmStructObject_MaxInlineBytes; + } + // Assert that the first and last bytes for the field agree on which side of + // the inline/outline boundary they live. + MOZ_RELEASE_ASSERT( + (fieldOffset < WasmStructObject_MaxInlineBytes) == + ((fieldOffset + fieldType.size() - 1) < WasmStructObject_MaxInlineBytes)); +} + +inline uint8_t* WasmStructObject::fieldOffsetToAddress(FieldType fieldType, + uint32_t fieldOffset) { + bool areaIsOutline; + uint32_t areaOffset; + fieldOffsetToAreaAndOffset(fieldType, fieldOffset, &areaIsOutline, + &areaOffset); + return ((uint8_t*)(areaIsOutline ? outlineData_ : &inlineData_[0])) + + areaOffset; +} + +// Ensure that faulting loads/stores for WasmStructObject and WasmArrayObject +// are in the NULL pointer guard page. +static_assert(WasmStructObject_MaxInlineBytes <= wasm::NullPtrGuardSize); +static_assert(sizeof(WasmArrayObject) <= wasm::NullPtrGuardSize); + +} // namespace js + +//========================================================================= +// misc + +namespace js { + +inline bool IsWasmGcObjectClass(const JSClass* class_) { + return class_ == &WasmArrayObject::class_ || + class_ == &WasmStructObject::classInline_ || + class_ == &WasmStructObject::classOutline_; +} + +} // namespace js + +template <> +inline bool JSObject::is() const { + return js::IsWasmGcObjectClass(getClass()); +} + +template <> +inline bool JSObject::is() const { + const JSClass* class_ = getClass(); + return class_ == &js::WasmStructObject::classInline_ || + class_ == &js::WasmStructObject::classOutline_; +} + +#endif /* wasm_WasmGcObject_h */ diff --git a/js/src/wasm/WasmGenerator.cpp b/js/src/wasm/WasmGenerator.cpp new file mode 100644 index 0000000000..503bbd70ee --- /dev/null +++ b/js/src/wasm/WasmGenerator.cpp @@ -0,0 +1,1279 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmGenerator.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/EnumeratedRange.h" +#include "mozilla/SHA1.h" + +#include + +#include "jit/Assembler.h" +#include "jit/JitOptions.h" +#include "js/Printf.h" +#include "threading/Thread.h" +#include "util/Memory.h" +#include "util/Text.h" +#include "vm/HelperThreads.h" +#include "vm/Time.h" +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmCompile.h" +#include "wasm/WasmGC.h" +#include "wasm/WasmIonCompile.h" +#include "wasm/WasmStubs.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::CheckedInt; +using mozilla::MakeEnumeratedRange; + +bool CompiledCode::swap(MacroAssembler& masm) { + MOZ_ASSERT(bytes.empty()); + if (!masm.swapBuffer(bytes)) { + return false; + } + + callSites.swap(masm.callSites()); + callSiteTargets.swap(masm.callSiteTargets()); + trapSites.swap(masm.trapSites()); + symbolicAccesses.swap(masm.symbolicAccesses()); + tryNotes.swap(masm.tryNotes()); + codeLabels.swap(masm.codeLabels()); + return true; +} + +// **************************************************************************** +// ModuleGenerator + +static const unsigned GENERATOR_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024; +static const unsigned COMPILATION_LIFO_DEFAULT_CHUNK_SIZE = 64 * 1024; +static const uint32_t BAD_CODE_RANGE = UINT32_MAX; + +ModuleGenerator::ModuleGenerator(const CompileArgs& args, + ModuleEnvironment* moduleEnv, + CompilerEnvironment* compilerEnv, + const Atomic* cancelled, + UniqueChars* error, + UniqueCharsVector* warnings) + : compileArgs_(&args), + error_(error), + warnings_(warnings), + cancelled_(cancelled), + moduleEnv_(moduleEnv), + compilerEnv_(compilerEnv), + linkData_(nullptr), + metadataTier_(nullptr), + lifo_(GENERATOR_LIFO_DEFAULT_CHUNK_SIZE), + masmAlloc_(&lifo_), + masm_(masmAlloc_, *moduleEnv, /* limitedSize= */ false), + debugTrapCodeOffset_(), + lastPatchedCallSite_(0), + startOfUnpatchedCallsites_(0), + parallel_(false), + outstanding_(0), + currentTask_(nullptr), + batchedBytecode_(0), + finishedFuncDefs_(false) {} + +ModuleGenerator::~ModuleGenerator() { + MOZ_ASSERT_IF(finishedFuncDefs_, !batchedBytecode_); + MOZ_ASSERT_IF(finishedFuncDefs_, !currentTask_); + + if (parallel_) { + if (outstanding_) { + AutoLockHelperThreadState lock; + + // Remove any pending compilation tasks from the worklist. + size_t removed = RemovePendingWasmCompileTasks(taskState_, mode(), lock); + MOZ_ASSERT(outstanding_ >= removed); + outstanding_ -= removed; + + // Wait until all active compilation tasks have finished. + while (true) { + MOZ_ASSERT(outstanding_ >= taskState_.finished().length()); + outstanding_ -= taskState_.finished().length(); + taskState_.finished().clear(); + + MOZ_ASSERT(outstanding_ >= taskState_.numFailed()); + outstanding_ -= taskState_.numFailed(); + taskState_.numFailed() = 0; + + if (!outstanding_) { + break; + } + + taskState_.condVar().wait(lock); /* failed or finished */ + } + } + } else { + MOZ_ASSERT(!outstanding_); + } + + // Propagate error state. + if (error_ && !*error_) { + AutoLockHelperThreadState lock; + *error_ = std::move(taskState_.errorMessage()); + } +} + +// This is the highest offset into Instance::globalArea that will not overflow +// a signed 32-bit integer. +static const uint32_t MaxInstanceDataOffset = + INT32_MAX - Instance::offsetOfData(); + +bool ModuleGenerator::allocateInstanceDataBytes(uint32_t bytes, uint32_t align, + uint32_t* instanceDataOffset) { + CheckedInt newInstanceDataLength(metadata_->instanceDataLength); + + // Adjust the current global data length so that it's aligned to `align` + newInstanceDataLength += + ComputeByteAlignment(newInstanceDataLength.value(), align); + if (!newInstanceDataLength.isValid()) { + return false; + } + + // The allocated data is given by the aligned length + *instanceDataOffset = newInstanceDataLength.value(); + + // Advance the length for `bytes` being allocated + newInstanceDataLength += bytes; + if (!newInstanceDataLength.isValid()) { + return false; + } + + // Check that the highest offset into this allocated space would not overflow + // a signed 32-bit integer. + if (newInstanceDataLength.value() > MaxInstanceDataOffset + 1) { + return false; + } + + metadata_->instanceDataLength = newInstanceDataLength.value(); + return true; +} + +bool ModuleGenerator::allocateInstanceDataBytesN(uint32_t bytes, uint32_t align, + uint32_t count, + uint32_t* instanceDataOffset) { + // The size of each allocation should be a multiple of alignment so that a + // contiguous array of allocations will be aligned + MOZ_ASSERT(bytes % align == 0); + + // Compute the total bytes being allocated + CheckedInt totalBytes = bytes; + totalBytes *= count; + if (!totalBytes.isValid()) { + return false; + } + + // Allocate the bytes + return allocateInstanceDataBytes(totalBytes.value(), align, + instanceDataOffset); +} + +bool ModuleGenerator::init(Metadata* maybeAsmJSMetadata) { + // Perform fallible metadata, linkdata, assumption allocations. + + MOZ_ASSERT(isAsmJS() == !!maybeAsmJSMetadata); + if (maybeAsmJSMetadata) { + metadata_ = maybeAsmJSMetadata; + } else { + metadata_ = js_new(); + if (!metadata_) { + return false; + } + } + + if (compileArgs_->scriptedCaller.filename) { + metadata_->filename = + DuplicateString(compileArgs_->scriptedCaller.filename.get()); + if (!metadata_->filename) { + return false; + } + + metadata_->filenameIsURL = compileArgs_->scriptedCaller.filenameIsURL; + } else { + MOZ_ASSERT(!compileArgs_->scriptedCaller.filenameIsURL); + } + + if (compileArgs_->sourceMapURL) { + metadata_->sourceMapURL = DuplicateString(compileArgs_->sourceMapURL.get()); + if (!metadata_->sourceMapURL) { + return false; + } + } + + linkData_ = js::MakeUnique(tier()); + if (!linkData_) { + return false; + } + + metadataTier_ = js::MakeUnique(tier()); + if (!metadataTier_) { + return false; + } + + // funcToCodeRange maps function indices to code-range indices and all + // elements will be initialized by the time module generation is finished. + + if (!metadataTier_->funcToCodeRange.appendN(BAD_CODE_RANGE, + moduleEnv_->funcs.length())) { + return false; + } + + // Pre-reserve space for large Vectors to avoid the significant cost of the + // final reallocs. In particular, the MacroAssembler can be enormous, so be + // extra conservative. Since large over-reservations may fail when the + // actual allocations will succeed, ignore OOM failures. Note, + // shrinkStorageToFit calls at the end will trim off unneeded capacity. + + size_t codeSectionSize = + moduleEnv_->codeSection ? moduleEnv_->codeSection->size : 0; + + size_t estimatedCodeSize = + size_t(1.2 * EstimateCompiledCodeSize(tier(), codeSectionSize)); + (void)masm_.reserve(std::min(estimatedCodeSize, MaxCodeBytesPerProcess)); + + (void)metadataTier_->codeRanges.reserve(2 * moduleEnv_->numFuncDefs()); + + const size_t ByteCodesPerCallSite = 50; + (void)metadataTier_->callSites.reserve(codeSectionSize / + ByteCodesPerCallSite); + + const size_t ByteCodesPerOOBTrap = 10; + (void)metadataTier_->trapSites[Trap::OutOfBounds].reserve( + codeSectionSize / ByteCodesPerOOBTrap); + + // Allocate space in instance for declarations that need it + MOZ_ASSERT(metadata_->instanceDataLength == 0); + + // Allocate space for type definitions + if (!allocateInstanceDataBytesN( + sizeof(TypeDefInstanceData), alignof(TypeDefInstanceData), + moduleEnv_->types->length(), &moduleEnv_->typeDefsOffsetStart)) { + return false; + } + metadata_->typeDefsOffsetStart = moduleEnv_->typeDefsOffsetStart; + + // Allocate space for every function import + if (!allocateInstanceDataBytesN( + sizeof(FuncImportInstanceData), alignof(FuncImportInstanceData), + moduleEnv_->numFuncImports, &moduleEnv_->funcImportsOffsetStart)) { + return false; + } + + // Allocate space for every table + if (!allocateInstanceDataBytesN( + sizeof(TableInstanceData), alignof(TableInstanceData), + moduleEnv_->tables.length(), &moduleEnv_->tablesOffsetStart)) { + return false; + } + metadata_->tablesOffsetStart = moduleEnv_->tablesOffsetStart; + + // Allocate space for every tag + if (!allocateInstanceDataBytesN( + sizeof(TagInstanceData), alignof(TagInstanceData), + moduleEnv_->tags.length(), &moduleEnv_->tagsOffsetStart)) { + return false; + } + metadata_->tagsOffsetStart = moduleEnv_->tagsOffsetStart; + + // Allocate space for every global that requires it + for (GlobalDesc& global : moduleEnv_->globals) { + if (global.isConstant()) { + continue; + } + + uint32_t width = global.isIndirect() ? sizeof(void*) : global.type().size(); + + uint32_t instanceDataOffset; + if (!allocateInstanceDataBytes(width, width, &instanceDataOffset)) { + return false; + } + + global.setOffset(instanceDataOffset); + } + + // Initialize function import metadata + if (!metadataTier_->funcImports.resize(moduleEnv_->numFuncImports)) { + return false; + } + + for (size_t i = 0; i < moduleEnv_->numFuncImports; i++) { + metadataTier_->funcImports[i] = + FuncImport(moduleEnv_->funcs[i].typeIndex, + moduleEnv_->offsetOfFuncImportInstanceData(i)); + } + + // Share type definitions with metadata + metadata_->types = moduleEnv_->types; + + // Accumulate all exported functions: + // - explicitly marked as such; + // - implicitly exported by being an element of function tables; + // - implicitly exported by being the start function; + // - implicitly exported by being used in global ref.func initializer + // ModuleEnvironment accumulates this information for us during decoding, + // transfer it to the FuncExportVector stored in Metadata. + + uint32_t exportedFuncCount = 0; + for (const FuncDesc& func : moduleEnv_->funcs) { + if (func.isExported()) { + exportedFuncCount++; + } + } + if (!metadataTier_->funcExports.reserve(exportedFuncCount)) { + return false; + } + + for (uint32_t funcIndex = 0; funcIndex < moduleEnv_->funcs.length(); + funcIndex++) { + const FuncDesc& func = moduleEnv_->funcs[funcIndex]; + + if (!func.isExported()) { + continue; + } + + metadataTier_->funcExports.infallibleEmplaceBack( + FuncExport(func.typeIndex, funcIndex, func.isEager())); + } + + // Determine whether parallel or sequential compilation is to be used and + // initialize the CompileTasks that will be used in either mode. + + MOZ_ASSERT(GetHelperThreadCount() > 1); + + uint32_t numTasks; + if (CanUseExtraThreads() && GetHelperThreadCPUCount() > 1) { + parallel_ = true; + numTasks = 2 * GetMaxWasmCompilationThreads(); + } else { + numTasks = 1; + } + + if (!tasks_.initCapacity(numTasks)) { + return false; + } + for (size_t i = 0; i < numTasks; i++) { + tasks_.infallibleEmplaceBack(*moduleEnv_, *compilerEnv_, taskState_, + COMPILATION_LIFO_DEFAULT_CHUNK_SIZE); + } + + if (!freeTasks_.reserve(numTasks)) { + return false; + } + for (size_t i = 0; i < numTasks; i++) { + freeTasks_.infallibleAppend(&tasks_[i]); + } + + // Fill in function stubs for each import so that imported functions can be + // used in all the places that normal function definitions can (table + // elements, export calls, etc). + + CompiledCode& importCode = tasks_[0].output; + MOZ_ASSERT(importCode.empty()); + + if (!GenerateImportFunctions(*moduleEnv_, metadataTier_->funcImports, + &importCode)) { + return false; + } + + if (!linkCompiledCode(importCode)) { + return false; + } + + importCode.clear(); + return true; +} + +bool ModuleGenerator::funcIsCompiled(uint32_t funcIndex) const { + return metadataTier_->funcToCodeRange[funcIndex] != BAD_CODE_RANGE; +} + +const CodeRange& ModuleGenerator::funcCodeRange(uint32_t funcIndex) const { + MOZ_ASSERT(funcIsCompiled(funcIndex)); + const CodeRange& cr = + metadataTier_->codeRanges[metadataTier_->funcToCodeRange[funcIndex]]; + MOZ_ASSERT(cr.isFunction()); + return cr; +} + +static bool InRange(uint32_t caller, uint32_t callee) { + // We assume JumpImmediateRange is defined conservatively enough that the + // slight difference between 'caller' (which is really the return address + // offset) and the actual base of the relative displacement computation + // isn't significant. + uint32_t range = std::min(JitOptions.jumpThreshold, JumpImmediateRange); + if (caller < callee) { + return callee - caller < range; + } + return caller - callee < range; +} + +using OffsetMap = + HashMap, SystemAllocPolicy>; +using TrapMaybeOffsetArray = + EnumeratedArray>; + +bool ModuleGenerator::linkCallSites() { + AutoCreatedBy acb(masm_, "linkCallSites"); + + masm_.haltingAlign(CodeAlignment); + + // Create far jumps for calls that have relative offsets that may otherwise + // go out of range. This method is called both between function bodies (at a + // frequency determined by the ISA's jump range) and once at the very end of + // a module's codegen after all possible calls/traps have been emitted. + + OffsetMap existingCallFarJumps; + for (; lastPatchedCallSite_ < metadataTier_->callSites.length(); + lastPatchedCallSite_++) { + const CallSite& callSite = metadataTier_->callSites[lastPatchedCallSite_]; + const CallSiteTarget& target = callSiteTargets_[lastPatchedCallSite_]; + uint32_t callerOffset = callSite.returnAddressOffset(); + switch (callSite.kind()) { + case CallSiteDesc::Import: + case CallSiteDesc::Indirect: + case CallSiteDesc::IndirectFast: + case CallSiteDesc::Symbolic: + case CallSiteDesc::Breakpoint: + case CallSiteDesc::EnterFrame: + case CallSiteDesc::LeaveFrame: + case CallSiteDesc::FuncRef: + case CallSiteDesc::FuncRefFast: + break; + case CallSiteDesc::Func: { + if (funcIsCompiled(target.funcIndex())) { + uint32_t calleeOffset = + funcCodeRange(target.funcIndex()).funcUncheckedCallEntry(); + if (InRange(callerOffset, calleeOffset)) { + masm_.patchCall(callerOffset, calleeOffset); + break; + } + } + + OffsetMap::AddPtr p = + existingCallFarJumps.lookupForAdd(target.funcIndex()); + if (!p) { + Offsets offsets; + offsets.begin = masm_.currentOffset(); + if (!callFarJumps_.emplaceBack(target.funcIndex(), + masm_.farJumpWithPatch())) { + return false; + } + offsets.end = masm_.currentOffset(); + if (masm_.oom()) { + return false; + } + if (!metadataTier_->codeRanges.emplaceBack(CodeRange::FarJumpIsland, + offsets)) { + return false; + } + if (!existingCallFarJumps.add(p, target.funcIndex(), offsets.begin)) { + return false; + } + } + + masm_.patchCall(callerOffset, p->value()); + break; + } + } + } + + masm_.flushBuffer(); + return !masm_.oom(); +} + +void ModuleGenerator::noteCodeRange(uint32_t codeRangeIndex, + const CodeRange& codeRange) { + switch (codeRange.kind()) { + case CodeRange::Function: + MOZ_ASSERT(metadataTier_->funcToCodeRange[codeRange.funcIndex()] == + BAD_CODE_RANGE); + metadataTier_->funcToCodeRange[codeRange.funcIndex()] = codeRangeIndex; + break; + case CodeRange::InterpEntry: + metadataTier_->lookupFuncExport(codeRange.funcIndex()) + .initEagerInterpEntryOffset(codeRange.begin()); + break; + case CodeRange::JitEntry: + // Nothing to do: jit entries are linked in the jump tables. + break; + case CodeRange::ImportJitExit: + metadataTier_->funcImports[codeRange.funcIndex()].initJitExitOffset( + codeRange.begin()); + break; + case CodeRange::ImportInterpExit: + metadataTier_->funcImports[codeRange.funcIndex()].initInterpExitOffset( + codeRange.begin()); + break; + case CodeRange::DebugTrap: + MOZ_ASSERT(!debugTrapCodeOffset_); + debugTrapCodeOffset_ = codeRange.begin(); + break; + case CodeRange::TrapExit: + MOZ_ASSERT(!linkData_->trapOffset); + linkData_->trapOffset = codeRange.begin(); + break; + case CodeRange::Throw: + // Jumped to by other stubs, so nothing to do. + break; + case CodeRange::FarJumpIsland: + case CodeRange::BuiltinThunk: + MOZ_CRASH("Unexpected CodeRange kind"); + } +} + +// Append every element from `srcVec` where `filterOp(srcElem) == true`. +// Applies `mutateOp(dstElem)` to every element that is appended. +template +static bool AppendForEach(Vec* dstVec, const Vec& srcVec, FilterOp filterOp, + MutateOp mutateOp) { + // Eagerly grow the vector to the whole src vector. Any filtered elements + // will be trimmed later. + if (!dstVec->growByUninitialized(srcVec.length())) { + return false; + } + + using T = typename Vec::ElementType; + + T* dstBegin = dstVec->begin(); + T* dstEnd = dstVec->end(); + + // We appended srcVec.length() elements at the beginning, so we append + // elements starting at the first uninitialized element. + T* dst = dstEnd - srcVec.length(); + + for (const T* src = srcVec.begin(); src != srcVec.end(); src++) { + if (!filterOp(src)) { + continue; + } + new (dst) T(*src); + mutateOp(dst - dstBegin, dst); + dst++; + } + + // Trim off the filtered out elements that were eagerly added at the + // beginning + size_t newSize = dst - dstBegin; + if (newSize != dstVec->length()) { + dstVec->shrinkTo(newSize); + } + + return true; +} + +template +bool FilterNothing(const T* element) { + return true; +} + +// The same as the above `AppendForEach`, without performing any filtering. +template +static bool AppendForEach(Vec* dstVec, const Vec& srcVec, MutateOp mutateOp) { + using T = typename Vec::ElementType; + return AppendForEach(dstVec, srcVec, &FilterNothing, mutateOp); +} + +bool ModuleGenerator::linkCompiledCode(CompiledCode& code) { + AutoCreatedBy acb(masm_, "ModuleGenerator::linkCompiledCode"); + JitContext jcx; + + // Before merging in new code, if calls in a prior code range might go out of + // range, insert far jumps to extend the range. + + if (!InRange(startOfUnpatchedCallsites_, + masm_.size() + code.bytes.length())) { + startOfUnpatchedCallsites_ = masm_.size(); + if (!linkCallSites()) { + return false; + } + } + + // All code offsets in 'code' must be incremented by their position in the + // overall module when the code was appended. + + masm_.haltingAlign(CodeAlignment); + const size_t offsetInModule = masm_.size(); + if (!masm_.appendRawCode(code.bytes.begin(), code.bytes.length())) { + return false; + } + + auto codeRangeOp = [offsetInModule, this](uint32_t codeRangeIndex, + CodeRange* codeRange) { + codeRange->offsetBy(offsetInModule); + noteCodeRange(codeRangeIndex, *codeRange); + }; + if (!AppendForEach(&metadataTier_->codeRanges, code.codeRanges, + codeRangeOp)) { + return false; + } + + auto callSiteOp = [=](uint32_t, CallSite* cs) { + cs->offsetBy(offsetInModule); + }; + if (!AppendForEach(&metadataTier_->callSites, code.callSites, callSiteOp)) { + return false; + } + + if (!callSiteTargets_.appendAll(code.callSiteTargets)) { + return false; + } + + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + auto trapSiteOp = [=](uint32_t, TrapSite* ts) { + ts->offsetBy(offsetInModule); + }; + if (!AppendForEach(&metadataTier_->trapSites[trap], code.trapSites[trap], + trapSiteOp)) { + return false; + } + } + + for (const SymbolicAccess& access : code.symbolicAccesses) { + uint32_t patchAt = offsetInModule + access.patchAt.offset(); + if (!linkData_->symbolicLinks[access.target].append(patchAt)) { + return false; + } + } + + for (const CodeLabel& codeLabel : code.codeLabels) { + LinkData::InternalLink link; + link.patchAtOffset = offsetInModule + codeLabel.patchAt().offset(); + link.targetOffset = offsetInModule + codeLabel.target().offset(); +#ifdef JS_CODELABEL_LINKMODE + link.mode = codeLabel.linkMode(); +#endif + if (!linkData_->internalLinks.append(link)) { + return false; + } + } + + for (size_t i = 0; i < code.stackMaps.length(); i++) { + StackMaps::Maplet maplet = code.stackMaps.move(i); + maplet.offsetBy(offsetInModule); + if (!metadataTier_->stackMaps.add(maplet)) { + // This function is now the only owner of maplet.map, so we'd better + // free it right now. + maplet.map->destroy(); + return false; + } + } + + auto tryNoteFilter = [](const TryNote* tn) { + // Filter out all try notes that were never given a try body. This may + // happen due to dead code elimination. + return tn->hasTryBody(); + }; + auto tryNoteOp = [=](uint32_t, TryNote* tn) { tn->offsetBy(offsetInModule); }; + return AppendForEach(&metadataTier_->tryNotes, code.tryNotes, tryNoteFilter, + tryNoteOp); +} + +static bool ExecuteCompileTask(CompileTask* task, UniqueChars* error) { + MOZ_ASSERT(task->lifo.isEmpty()); + MOZ_ASSERT(task->output.empty()); + + switch (task->compilerEnv.tier()) { + case Tier::Optimized: + if (!IonCompileFunctions(task->moduleEnv, task->compilerEnv, task->lifo, + task->inputs, &task->output, error)) { + return false; + } + break; + case Tier::Baseline: + if (!BaselineCompileFunctions(task->moduleEnv, task->compilerEnv, + task->lifo, task->inputs, &task->output, + error)) { + return false; + } + break; + } + + MOZ_ASSERT(task->lifo.isEmpty()); + MOZ_ASSERT(task->inputs.length() == task->output.codeRanges.length()); + task->inputs.clear(); + return true; +} + +void CompileTask::runHelperThreadTask(AutoLockHelperThreadState& lock) { + UniqueChars error; + bool ok; + + { + AutoUnlockHelperThreadState unlock(lock); + ok = ExecuteCompileTask(this, &error); + } + + // Don't release the lock between updating our state and returning from this + // method. + + if (!ok || !state.finished().append(this)) { + state.numFailed()++; + if (!state.errorMessage()) { + state.errorMessage() = std::move(error); + } + } + + state.condVar().notify_one(); /* failed or finished */ +} + +ThreadType CompileTask::threadType() { + switch (compilerEnv.mode()) { + case CompileMode::Once: + case CompileMode::Tier1: + return ThreadType::THREAD_TYPE_WASM_COMPILE_TIER1; + case CompileMode::Tier2: + return ThreadType::THREAD_TYPE_WASM_COMPILE_TIER2; + default: + MOZ_CRASH(); + } +} + +bool ModuleGenerator::locallyCompileCurrentTask() { + if (!ExecuteCompileTask(currentTask_, error_)) { + return false; + } + if (!finishTask(currentTask_)) { + return false; + } + currentTask_ = nullptr; + batchedBytecode_ = 0; + return true; +} + +bool ModuleGenerator::finishTask(CompileTask* task) { + AutoCreatedBy acb(masm_, "ModuleGenerator::finishTask"); + + masm_.haltingAlign(CodeAlignment); + + if (!linkCompiledCode(task->output)) { + return false; + } + + task->output.clear(); + + MOZ_ASSERT(task->inputs.empty()); + MOZ_ASSERT(task->output.empty()); + MOZ_ASSERT(task->lifo.isEmpty()); + freeTasks_.infallibleAppend(task); + return true; +} + +bool ModuleGenerator::launchBatchCompile() { + MOZ_ASSERT(currentTask_); + + if (cancelled_ && *cancelled_) { + return false; + } + + if (!parallel_) { + return locallyCompileCurrentTask(); + } + + if (!StartOffThreadWasmCompile(currentTask_, mode())) { + return false; + } + outstanding_++; + currentTask_ = nullptr; + batchedBytecode_ = 0; + return true; +} + +bool ModuleGenerator::finishOutstandingTask() { + MOZ_ASSERT(parallel_); + + CompileTask* task = nullptr; + { + AutoLockHelperThreadState lock; + while (true) { + MOZ_ASSERT(outstanding_ > 0); + + if (taskState_.numFailed() > 0) { + return false; + } + + if (!taskState_.finished().empty()) { + outstanding_--; + task = taskState_.finished().popCopy(); + break; + } + + taskState_.condVar().wait(lock); /* failed or finished */ + } + } + + // Call outside of the compilation lock. + return finishTask(task); +} + +bool ModuleGenerator::compileFuncDef(uint32_t funcIndex, + uint32_t lineOrBytecode, + const uint8_t* begin, const uint8_t* end, + Uint32Vector&& lineNums) { + MOZ_ASSERT(!finishedFuncDefs_); + MOZ_ASSERT(funcIndex < moduleEnv_->numFuncs()); + + uint32_t threshold; + switch (tier()) { + case Tier::Baseline: + threshold = JitOptions.wasmBatchBaselineThreshold; + break; + case Tier::Optimized: + threshold = JitOptions.wasmBatchIonThreshold; + break; + default: + MOZ_CRASH("Invalid tier value"); + break; + } + + uint32_t funcBytecodeLength = end - begin; + + // Do not go over the threshold if we can avoid it: spin off the compilation + // before appending the function if we would go over. (Very large single + // functions may still exceed the threshold but this is fine; it'll be very + // uncommon and is in any case safely handled by the MacroAssembler's buffer + // limit logic.) + + if (currentTask_ && currentTask_->inputs.length() && + batchedBytecode_ + funcBytecodeLength > threshold) { + if (!launchBatchCompile()) { + return false; + } + } + + if (!currentTask_) { + if (freeTasks_.empty() && !finishOutstandingTask()) { + return false; + } + currentTask_ = freeTasks_.popCopy(); + } + + if (!currentTask_->inputs.emplaceBack(funcIndex, lineOrBytecode, begin, end, + std::move(lineNums))) { + return false; + } + + batchedBytecode_ += funcBytecodeLength; + MOZ_ASSERT(batchedBytecode_ <= MaxCodeSectionBytes); + return true; +} + +bool ModuleGenerator::finishFuncDefs() { + MOZ_ASSERT(!finishedFuncDefs_); + + if (currentTask_ && !locallyCompileCurrentTask()) { + return false; + } + + finishedFuncDefs_ = true; + return true; +} + +bool ModuleGenerator::finishCodegen() { + // Now that all functions and stubs are generated and their CodeRanges + // known, patch all calls (which can emit far jumps) and far jumps. Linking + // can emit tiny far-jump stubs, so there is an ordering dependency here. + + if (!linkCallSites()) { + return false; + } + + for (CallFarJump far : callFarJumps_) { + masm_.patchFarJump(far.jump, + funcCodeRange(far.funcIndex).funcUncheckedCallEntry()); + } + + metadataTier_->debugTrapOffset = debugTrapCodeOffset_; + + // None of the linking or far-jump operations should emit masm metadata. + + MOZ_ASSERT(masm_.callSites().empty()); + MOZ_ASSERT(masm_.callSiteTargets().empty()); + MOZ_ASSERT(masm_.trapSites().empty()); + MOZ_ASSERT(masm_.symbolicAccesses().empty()); + MOZ_ASSERT(masm_.tryNotes().empty()); + MOZ_ASSERT(masm_.codeLabels().empty()); + + masm_.finish(); + return !masm_.oom(); +} + +bool ModuleGenerator::finishMetadataTier() { + // The stackmaps aren't yet sorted. Do so now, since we'll need to + // binary-search them at GC time. + metadataTier_->stackMaps.finishAndSort(); + + // The try notes also need to be sorted to simplify lookup. + std::sort(metadataTier_->tryNotes.begin(), metadataTier_->tryNotes.end()); + +#ifdef DEBUG + // Check that the stackmap contains no duplicates, since that could lead to + // ambiguities about stack slot pointerness. + const uint8_t* previousNextInsnAddr = nullptr; + for (size_t i = 0; i < metadataTier_->stackMaps.length(); i++) { + const StackMaps::Maplet& maplet = metadataTier_->stackMaps.get(i); + MOZ_ASSERT_IF(i > 0, uintptr_t(maplet.nextInsnAddr) > + uintptr_t(previousNextInsnAddr)); + previousNextInsnAddr = maplet.nextInsnAddr; + } + + // Assert all sorted metadata is sorted. + uint32_t last = 0; + for (const CodeRange& codeRange : metadataTier_->codeRanges) { + MOZ_ASSERT(codeRange.begin() >= last); + last = codeRange.end(); + } + + last = 0; + for (const CallSite& callSite : metadataTier_->callSites) { + MOZ_ASSERT(callSite.returnAddressOffset() >= last); + last = callSite.returnAddressOffset(); + } + + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + last = 0; + for (const TrapSite& trapSite : metadataTier_->trapSites[trap]) { + MOZ_ASSERT(trapSite.pcOffset >= last); + last = trapSite.pcOffset; + } + } + + // Try notes should be sorted so that the end of ranges are in rising order + // so that the innermost catch handler is chosen. + last = 0; + for (const TryNote& tryNote : metadataTier_->tryNotes) { + MOZ_ASSERT(tryNote.tryBodyEnd() >= last); + MOZ_ASSERT(tryNote.tryBodyEnd() > tryNote.tryBodyBegin()); + last = tryNote.tryBodyBegin(); + } +#endif + + // These Vectors can get large and the excess capacity can be significant, + // so realloc them down to size. + + metadataTier_->funcToCodeRange.shrinkStorageToFit(); + metadataTier_->codeRanges.shrinkStorageToFit(); + metadataTier_->callSites.shrinkStorageToFit(); + metadataTier_->trapSites.shrinkStorageToFit(); + metadataTier_->tryNotes.shrinkStorageToFit(); + for (Trap trap : MakeEnumeratedRange(Trap::Limit)) { + metadataTier_->trapSites[trap].shrinkStorageToFit(); + } + + return true; +} + +UniqueCodeTier ModuleGenerator::finishCodeTier() { + MOZ_ASSERT(finishedFuncDefs_); + + while (outstanding_ > 0) { + if (!finishOutstandingTask()) { + return nullptr; + } + } + +#ifdef DEBUG + for (uint32_t codeRangeIndex : metadataTier_->funcToCodeRange) { + MOZ_ASSERT(codeRangeIndex != BAD_CODE_RANGE); + } +#endif + + // Now that all imports/exports are known, we can generate a special + // CompiledCode containing stubs. + + CompiledCode& stubCode = tasks_[0].output; + MOZ_ASSERT(stubCode.empty()); + + if (!GenerateStubs(*moduleEnv_, metadataTier_->funcImports, + metadataTier_->funcExports, &stubCode)) { + return nullptr; + } + + if (!linkCompiledCode(stubCode)) { + return nullptr; + } + + // Finish linking and metadata. + + if (!finishCodegen()) { + return nullptr; + } + + if (!finishMetadataTier()) { + return nullptr; + } + + UniqueModuleSegment segment = + ModuleSegment::create(tier(), masm_, *linkData_); + if (!segment) { + warnf("failed to allocate executable memory for module"); + return nullptr; + } + + metadataTier_->stackMaps.offsetBy(uintptr_t(segment->base())); + +#ifdef DEBUG + // Check that each stackmap is associated with a plausible instruction. + for (size_t i = 0; i < metadataTier_->stackMaps.length(); i++) { + MOZ_ASSERT(IsValidStackMapKey(compilerEnv_->debugEnabled(), + metadataTier_->stackMaps.get(i).nextInsnAddr), + "wasm stackmap does not reference a valid insn"); + } +#endif + + return js::MakeUnique(std::move(metadataTier_), std::move(segment)); +} + +SharedMetadata ModuleGenerator::finishMetadata(const Bytes& bytecode) { + // Finish initialization of Metadata, which is only needed for constructing + // the initial Module, not for tier-2 compilation. + MOZ_ASSERT(mode() != CompileMode::Tier2); + + // Copy over data from the ModuleEnvironment. + + metadata_->memory = moduleEnv_->memory; + metadata_->startFuncIndex = moduleEnv_->startFuncIndex; + metadata_->tables = std::move(moduleEnv_->tables); + metadata_->globals = std::move(moduleEnv_->globals); + metadata_->tags = std::move(moduleEnv_->tags); + metadata_->nameCustomSectionIndex = moduleEnv_->nameCustomSectionIndex; + metadata_->moduleName = moduleEnv_->moduleName; + metadata_->funcNames = std::move(moduleEnv_->funcNames); + metadata_->omitsBoundsChecks = moduleEnv_->hugeMemoryEnabled(); + + // Copy over additional debug information. + + if (compilerEnv_->debugEnabled()) { + metadata_->debugEnabled = true; + + const size_t numFuncs = moduleEnv_->funcs.length(); + if (!metadata_->debugFuncTypeIndices.resize(numFuncs)) { + return nullptr; + } + for (size_t i = 0; i < numFuncs; i++) { + metadata_->debugFuncTypeIndices[i] = moduleEnv_->funcs[i].typeIndex; + } + + static_assert(sizeof(ModuleHash) <= sizeof(mozilla::SHA1Sum::Hash), + "The ModuleHash size shall not exceed the SHA1 hash size."); + mozilla::SHA1Sum::Hash hash; + mozilla::SHA1Sum sha1Sum; + sha1Sum.update(bytecode.begin(), bytecode.length()); + sha1Sum.finish(hash); + memcpy(metadata_->debugHash, hash, sizeof(ModuleHash)); + } + + MOZ_ASSERT_IF(moduleEnv_->nameCustomSectionIndex, !!metadata_->namePayload); + + // Metadata shouldn't be mutably modified after finishMetadata(). + SharedMetadata metadata = metadata_; + metadata_ = nullptr; + return metadata; +} + +SharedModule ModuleGenerator::finishModule( + const ShareableBytes& bytecode, + JS::OptimizedEncodingListener* maybeTier2Listener) { + MOZ_ASSERT(mode() == CompileMode::Once || mode() == CompileMode::Tier1); + + UniqueCodeTier codeTier = finishCodeTier(); + if (!codeTier) { + return nullptr; + } + + JumpTables jumpTables; + if (!jumpTables.init(mode(), codeTier->segment(), + codeTier->metadata().codeRanges)) { + return nullptr; + } + + // Copy over data from the Bytecode, which is going away at the end of + // compilation. + + DataSegmentVector dataSegments; + if (!dataSegments.reserve(moduleEnv_->dataSegments.length())) { + return nullptr; + } + for (const DataSegmentEnv& srcSeg : moduleEnv_->dataSegments) { + MutableDataSegment dstSeg = js_new(); + if (!dstSeg) { + return nullptr; + } + if (!dstSeg->init(bytecode, srcSeg)) { + return nullptr; + } + dataSegments.infallibleAppend(std::move(dstSeg)); + } + + CustomSectionVector customSections; + if (!customSections.reserve(moduleEnv_->customSections.length())) { + return nullptr; + } + for (const CustomSectionEnv& srcSec : moduleEnv_->customSections) { + CustomSection sec; + if (!sec.name.append(bytecode.begin() + srcSec.nameOffset, + srcSec.nameLength)) { + return nullptr; + } + MutableBytes payload = js_new(); + if (!payload) { + return nullptr; + } + if (!payload->append(bytecode.begin() + srcSec.payloadOffset, + srcSec.payloadLength)) { + return nullptr; + } + sec.payload = std::move(payload); + customSections.infallibleAppend(std::move(sec)); + } + + if (moduleEnv_->nameCustomSectionIndex) { + metadata_->namePayload = + customSections[*moduleEnv_->nameCustomSectionIndex].payload; + } + + SharedMetadata metadata = finishMetadata(bytecode.bytes); + if (!metadata) { + return nullptr; + } + + MutableCode code = + js_new(std::move(codeTier), *metadata, std::move(jumpTables)); + if (!code || !code->initialize(*linkData_)) { + return nullptr; + } + + const ShareableBytes* debugBytecode = nullptr; + if (compilerEnv_->debugEnabled()) { + MOZ_ASSERT(mode() == CompileMode::Once); + MOZ_ASSERT(tier() == Tier::Debug); + debugBytecode = &bytecode; + } + + // All the components are finished, so create the complete Module and start + // tier-2 compilation if requested. + + MutableModule module = js_new( + *code, std::move(moduleEnv_->imports), std::move(moduleEnv_->exports), + std::move(dataSegments), std::move(moduleEnv_->elemSegments), + std::move(customSections), debugBytecode); + if (!module) { + return nullptr; + } + + if (!isAsmJS() && compileArgs_->features.testSerialization) { + MOZ_RELEASE_ASSERT(mode() == CompileMode::Once && + tier() == Tier::Serialized); + + Bytes serializedBytes; + if (!module->serialize(*linkData_, &serializedBytes)) { + return nullptr; + } + + MutableModule deserializedModule = + Module::deserialize(serializedBytes.begin(), serializedBytes.length()); + if (!deserializedModule) { + return nullptr; + } + module = deserializedModule; + + // Perform storeOptimizedEncoding here instead of below so we don't have to + // re-serialize the module. + if (maybeTier2Listener) { + maybeTier2Listener->storeOptimizedEncoding(serializedBytes.begin(), + serializedBytes.length()); + maybeTier2Listener = nullptr; + } + } + + if (mode() == CompileMode::Tier1) { + module->startTier2(*compileArgs_, bytecode, maybeTier2Listener); + } else if (tier() == Tier::Serialized && maybeTier2Listener) { + Bytes bytes; + if (module->serialize(*linkData_, &bytes)) { + maybeTier2Listener->storeOptimizedEncoding(bytes.begin(), bytes.length()); + } + } + + return module; +} + +bool ModuleGenerator::finishTier2(const Module& module) { + MOZ_ASSERT(mode() == CompileMode::Tier2); + MOZ_ASSERT(tier() == Tier::Optimized); + MOZ_ASSERT(!compilerEnv_->debugEnabled()); + + if (cancelled_ && *cancelled_) { + return false; + } + + UniqueCodeTier codeTier = finishCodeTier(); + if (!codeTier) { + return false; + } + + if (MOZ_UNLIKELY(JitOptions.wasmDelayTier2)) { + // Introduce an artificial delay when testing wasmDelayTier2, since we + // want to exercise both tier1 and tier2 code in this case. + ThisThread::SleepMilliseconds(500); + } + + return module.finishTier2(*linkData_, std::move(codeTier)); +} + +void ModuleGenerator::warnf(const char* msg, ...) { + if (!warnings_) { + return; + } + + va_list ap; + va_start(ap, msg); + UniqueChars str(JS_vsmprintf(msg, ap)); + va_end(ap); + if (!str) { + return; + } + + (void)warnings_->append(std::move(str)); +} + +size_t CompiledCode::sizeOfExcludingThis( + mozilla::MallocSizeOf mallocSizeOf) const { + size_t trapSitesSize = 0; + for (const TrapSiteVector& vec : trapSites) { + trapSitesSize += vec.sizeOfExcludingThis(mallocSizeOf); + } + + return bytes.sizeOfExcludingThis(mallocSizeOf) + + codeRanges.sizeOfExcludingThis(mallocSizeOf) + + callSites.sizeOfExcludingThis(mallocSizeOf) + + callSiteTargets.sizeOfExcludingThis(mallocSizeOf) + trapSitesSize + + symbolicAccesses.sizeOfExcludingThis(mallocSizeOf) + + tryNotes.sizeOfExcludingThis(mallocSizeOf) + + codeLabels.sizeOfExcludingThis(mallocSizeOf); +} + +size_t CompileTask::sizeOfExcludingThis( + mozilla::MallocSizeOf mallocSizeOf) const { + return lifo.sizeOfExcludingThis(mallocSizeOf) + + inputs.sizeOfExcludingThis(mallocSizeOf) + + output.sizeOfExcludingThis(mallocSizeOf); +} diff --git a/js/src/wasm/WasmGenerator.h b/js/src/wasm/WasmGenerator.h new file mode 100644 index 0000000000..032158b7a7 --- /dev/null +++ b/js/src/wasm/WasmGenerator.h @@ -0,0 +1,264 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_generator_h +#define wasm_generator_h + +#include "mozilla/Attributes.h" +#include "mozilla/MemoryReporting.h" + +#include "jit/MacroAssembler.h" +#include "threading/ProtectedData.h" +#include "vm/HelperThreadTask.h" +#include "wasm/WasmCompile.h" +#include "wasm/WasmModule.h" +#include "wasm/WasmValidate.h" + +namespace JS { +class OptimizedEncodingListener; +} + +namespace js { +namespace wasm { + +struct CompileTask; +using CompileTaskPtrVector = Vector; + +// FuncCompileInput contains the input for compiling a single function. + +struct FuncCompileInput { + const uint8_t* begin; + const uint8_t* end; + uint32_t index; + uint32_t lineOrBytecode; + Uint32Vector callSiteLineNums; + + FuncCompileInput(uint32_t index, uint32_t lineOrBytecode, + const uint8_t* begin, const uint8_t* end, + Uint32Vector&& callSiteLineNums) + : begin(begin), + end(end), + index(index), + lineOrBytecode(lineOrBytecode), + callSiteLineNums(std::move(callSiteLineNums)) {} +}; + +using FuncCompileInputVector = Vector; + +// CompiledCode contains the resulting code and metadata for a set of compiled +// input functions or stubs. + +struct CompiledCode { + Bytes bytes; + CodeRangeVector codeRanges; + CallSiteVector callSites; + CallSiteTargetVector callSiteTargets; + TrapSiteVectorArray trapSites; + SymbolicAccessVector symbolicAccesses; + jit::CodeLabelVector codeLabels; + StackMaps stackMaps; + TryNoteVector tryNotes; + + [[nodiscard]] bool swap(jit::MacroAssembler& masm); + + void clear() { + bytes.clear(); + codeRanges.clear(); + callSites.clear(); + callSiteTargets.clear(); + trapSites.clear(); + symbolicAccesses.clear(); + codeLabels.clear(); + stackMaps.clear(); + tryNotes.clear(); + MOZ_ASSERT(empty()); + } + + bool empty() { + return bytes.empty() && codeRanges.empty() && callSites.empty() && + callSiteTargets.empty() && trapSites.empty() && + symbolicAccesses.empty() && codeLabels.empty() && tryNotes.empty() && + stackMaps.empty(); + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +// The CompileTaskState of a ModuleGenerator contains the mutable state shared +// between helper threads executing CompileTasks. Each CompileTask started on a +// helper thread eventually either ends up in the 'finished' list or increments +// 'numFailed'. + +struct CompileTaskState { + HelperThreadLockData finished_; + HelperThreadLockData numFailed_; + HelperThreadLockData errorMessage_; + HelperThreadLockData condVar_; + + CompileTaskState() : numFailed_(0) {} + ~CompileTaskState() { + MOZ_ASSERT(finished_.refNoCheck().empty()); + MOZ_ASSERT(!numFailed_.refNoCheck()); + } + + CompileTaskPtrVector& finished() { return finished_.ref(); } + uint32_t& numFailed() { return numFailed_.ref(); } + UniqueChars& errorMessage() { return errorMessage_.ref(); } + ConditionVariable& condVar() { return condVar_.ref(); } +}; + +// A CompileTask holds a batch of input functions that are to be compiled on a +// helper thread as well as, eventually, the results of compilation. + +struct CompileTask : public HelperThreadTask { + const ModuleEnvironment& moduleEnv; + const CompilerEnvironment& compilerEnv; + + CompileTaskState& state; + LifoAlloc lifo; + FuncCompileInputVector inputs; + CompiledCode output; + + CompileTask(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, CompileTaskState& state, + size_t defaultChunkSize) + : moduleEnv(moduleEnv), + compilerEnv(compilerEnv), + state(state), + lifo(defaultChunkSize) {} + + virtual ~CompileTask() = default; + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + + void runHelperThreadTask(AutoLockHelperThreadState& locked) override; + ThreadType threadType() override; +}; + +// A ModuleGenerator encapsulates the creation of a wasm module. During the +// lifetime of a ModuleGenerator, a sequence of FunctionGenerators are created +// and destroyed to compile the individual function bodies. After generating all +// functions, ModuleGenerator::finish() must be called to complete the +// compilation and extract the resulting wasm module. + +class MOZ_STACK_CLASS ModuleGenerator { + using CompileTaskVector = Vector; + using CodeOffsetVector = Vector; + struct CallFarJump { + uint32_t funcIndex; + jit::CodeOffset jump; + CallFarJump(uint32_t fi, jit::CodeOffset j) : funcIndex(fi), jump(j) {} + }; + using CallFarJumpVector = Vector; + + // Constant parameters + SharedCompileArgs const compileArgs_; + UniqueChars* const error_; + UniqueCharsVector* const warnings_; + const Atomic* const cancelled_; + ModuleEnvironment* const moduleEnv_; + CompilerEnvironment* const compilerEnv_; + + // Data that is moved into the result of finish() + UniqueLinkData linkData_; + UniqueMetadataTier metadataTier_; + MutableMetadata metadata_; + + // Data scoped to the ModuleGenerator's lifetime + CompileTaskState taskState_; + LifoAlloc lifo_; + jit::TempAllocator masmAlloc_; + jit::WasmMacroAssembler masm_; + Uint32Vector funcToCodeRange_; + uint32_t debugTrapCodeOffset_; + CallFarJumpVector callFarJumps_; + CallSiteTargetVector callSiteTargets_; + uint32_t lastPatchedCallSite_; + uint32_t startOfUnpatchedCallsites_; + + // Parallel compilation + bool parallel_; + uint32_t outstanding_; + CompileTaskVector tasks_; + CompileTaskPtrVector freeTasks_; + CompileTask* currentTask_; + uint32_t batchedBytecode_; + + // Assertions + DebugOnly finishedFuncDefs_; + + bool allocateInstanceDataBytes(uint32_t bytes, uint32_t align, + uint32_t* instanceDataOffset); + bool allocateInstanceDataBytesN(uint32_t bytes, uint32_t align, + uint32_t count, uint32_t* instanceDataOffset); + + bool funcIsCompiled(uint32_t funcIndex) const; + const CodeRange& funcCodeRange(uint32_t funcIndex) const; + bool linkCallSites(); + void noteCodeRange(uint32_t codeRangeIndex, const CodeRange& codeRange); + bool linkCompiledCode(CompiledCode& code); + bool locallyCompileCurrentTask(); + bool finishTask(CompileTask* task); + bool launchBatchCompile(); + bool finishOutstandingTask(); + bool finishCodegen(); + bool finishMetadataTier(); + UniqueCodeTier finishCodeTier(); + SharedMetadata finishMetadata(const Bytes& bytecode); + + bool isAsmJS() const { return moduleEnv_->isAsmJS(); } + Tier tier() const { return compilerEnv_->tier(); } + CompileMode mode() const { return compilerEnv_->mode(); } + bool debugEnabled() const { return compilerEnv_->debugEnabled(); } + + void warnf(const char* msg, ...) MOZ_FORMAT_PRINTF(2, 3); + + public: + ModuleGenerator(const CompileArgs& args, ModuleEnvironment* moduleEnv, + CompilerEnvironment* compilerEnv, + const Atomic* cancelled, UniqueChars* error, + UniqueCharsVector* warnings); + ~ModuleGenerator(); + [[nodiscard]] bool init(Metadata* maybeAsmJSMetadata = nullptr); + + // Before finishFuncDefs() is called, compileFuncDef() must be called once + // for each funcIndex in the range [0, env->numFuncDefs()). + + [[nodiscard]] bool compileFuncDef( + uint32_t funcIndex, uint32_t lineOrBytecode, const uint8_t* begin, + const uint8_t* end, Uint32Vector&& callSiteLineNums = Uint32Vector()); + + // Must be called after the last compileFuncDef() and before finishModule() + // or finishTier2(). + + [[nodiscard]] bool finishFuncDefs(); + + // If env->mode is Once or Tier1, finishModule() must be called to generate + // a new Module. Otherwise, if env->mode is Tier2, finishTier2() must be + // called to augment the given Module with tier 2 code. + + SharedModule finishModule( + const ShareableBytes& bytecode, + JS::OptimizedEncodingListener* maybeTier2Listener = nullptr); + [[nodiscard]] bool finishTier2(const Module& module); +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_generator_h diff --git a/js/src/wasm/WasmInitExpr.cpp b/js/src/wasm/WasmInitExpr.cpp new file mode 100644 index 0000000000..91a9cb3b69 --- /dev/null +++ b/js/src/wasm/WasmInitExpr.cpp @@ -0,0 +1,664 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmInitExpr.h" + +#include "mozilla/Maybe.h" + +#include "js/Value.h" + +#include "wasm/WasmGcObject.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmOpIter.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmUtility.h" +#include "wasm/WasmValidate.h" + +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::wasm; + +static bool ValidateInitExpr(Decoder& d, ModuleEnvironment* env, + ValType expected, + uint32_t maxInitializedGlobalsIndexPlus1, + Maybe* literal) { + ValidatingOpIter iter(*env, d, ValidatingOpIter::InitExpr); + + if (!iter.startInitExpr(expected, maxInitializedGlobalsIndexPlus1)) { + return false; + } + + // Perform trivial constant recovery, this is done so that codegen may + // generate optimal code for global.get on immutable globals with simple + // initializers. + // + // We simply update the last seen literal value while validating an + // instruction with a literal value, and clear the literal value when + // validating an instruction with a dynamic value. The last value is the + // literal for this init expressions, if any. This is correct because there + // are no drops or control flow allowed in init expressions. + *literal = Nothing(); + + while (true) { + OpBytes op; + if (!iter.readOp(&op)) { + return false; + } + +#if defined(ENABLE_WASM_EXTENDED_CONST) || defined(ENABLE_WASM_GC) + Nothing nothing; +#endif + NothingVector nothings{}; + ResultType unusedType; + + switch (op.b0) { + case uint16_t(Op::End): { + LabelKind kind; + if (!iter.readEnd(&kind, &unusedType, ¬hings, ¬hings)) { + return false; + } + MOZ_ASSERT(kind == LabelKind::Body); + iter.popEnd(); + if (iter.controlStackEmpty()) { + return iter.endInitExpr(); + } + break; + } + case uint16_t(Op::GlobalGet): { + uint32_t index; + if (!iter.readGetGlobal(&index)) { + return false; + } + *literal = Nothing(); + break; + } + case uint16_t(Op::I32Const): { + int32_t c; + if (!iter.readI32Const(&c)) { + return false; + } + *literal = Some(LitVal(uint32_t(c))); + break; + } + case uint16_t(Op::I64Const): { + int64_t c; + if (!iter.readI64Const(&c)) { + return false; + } + *literal = Some(LitVal(uint64_t(c))); + break; + } + case uint16_t(Op::F32Const): { + float c; + if (!iter.readF32Const(&c)) { + return false; + } + *literal = Some(LitVal(c)); + break; + } + case uint16_t(Op::F64Const): { + double c; + if (!iter.readF64Const(&c)) { + return false; + } + *literal = Some(LitVal(c)); + break; + } +#ifdef ENABLE_WASM_SIMD + case uint16_t(Op::SimdPrefix): { + if (!env->simdAvailable()) { + return d.fail("v128 not enabled"); + } + if (op.b1 != uint32_t(SimdOp::V128Const)) { + return iter.unrecognizedOpcode(&op); + } + V128 c; + if (!iter.readV128Const(&c)) { + return false; + } + *literal = Some(LitVal(c)); + break; + } +#endif + case uint16_t(Op::RefFunc): { + uint32_t funcIndex; + if (!iter.readRefFunc(&funcIndex)) { + return false; + } + env->declareFuncExported(funcIndex, /* eager */ false, + /* canRefFunc */ true); + *literal = Nothing(); + break; + } + case uint16_t(Op::RefNull): { + RefType type; + if (!iter.readRefNull(&type)) { + return false; + } + *literal = Some(LitVal(ValType(type))); + break; + } +#ifdef ENABLE_WASM_EXTENDED_CONST + case uint16_t(Op::I32Add): + case uint16_t(Op::I32Sub): + case uint16_t(Op::I32Mul): { + if (!env->extendedConstEnabled()) { + return iter.unrecognizedOpcode(&op); + } + if (!iter.readBinary(ValType::I32, ¬hing, ¬hing)) { + return false; + } + *literal = Nothing(); + break; + } + case uint16_t(Op::I64Add): + case uint16_t(Op::I64Sub): + case uint16_t(Op::I64Mul): { + if (!env->extendedConstEnabled()) { + return iter.unrecognizedOpcode(&op); + } + if (!iter.readBinary(ValType::I64, ¬hing, ¬hing)) { + return false; + } + *literal = Nothing(); + break; + } +#endif +#ifdef ENABLE_WASM_GC + case uint16_t(Op::GcPrefix): { + if (!env->gcEnabled()) { + return iter.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(GcOp::StructNew): { + uint32_t typeIndex; + if (!iter.readStructNew(&typeIndex, ¬hings)) { + return false; + } + break; + } + case uint32_t(GcOp::StructNewDefault): { + uint32_t typeIndex; + if (!iter.readStructNewDefault(&typeIndex)) { + return false; + } + break; + } + case uint32_t(GcOp::ArrayNew): { + uint32_t typeIndex; + if (!iter.readArrayNew(&typeIndex, ¬hing, ¬hing)) { + return false; + } + break; + } + case uint32_t(GcOp::ArrayNewFixed): { + uint32_t typeIndex, len; + if (!iter.readArrayNewFixed(&typeIndex, &len, ¬hings)) { + return false; + } + break; + } + case uint32_t(GcOp::ArrayNewDefault): { + uint32_t typeIndex; + if (!iter.readArrayNewDefault(&typeIndex, ¬hing)) { + return false; + } + break; + } + default: { + return iter.unrecognizedOpcode(&op); + } + } + *literal = Nothing(); + break; + } +#endif + default: { + return iter.unrecognizedOpcode(&op); + } + } + } +} + +class MOZ_STACK_CLASS InitExprInterpreter { + public: + explicit InitExprInterpreter(JSContext* cx, + Handle instanceObj) + : features(FeatureArgs::build(cx, FeatureOptions())), + stack(cx), + instanceObj(cx, instanceObj), + types(instanceObj->instance().metadata().types) {} + + bool evaluate(JSContext* cx, Decoder& d); + + Val result() { + MOZ_ASSERT(stack.length() == 1); + return stack.popCopy(); + } + + private: + FeatureArgs features; + RootedValVectorN<48> stack; + Rooted instanceObj; + SharedTypeContext types; + + Instance& instance() { return instanceObj->instance(); } + + [[nodiscard]] bool pushI32(int32_t c) { + return stack.append(Val(uint32_t(c))); + } + [[nodiscard]] bool pushI64(int64_t c) { + return stack.append(Val(uint64_t(c))); + } + [[nodiscard]] bool pushF32(float c) { return stack.append(Val(c)); } + [[nodiscard]] bool pushF64(double c) { return stack.append(Val(c)); } + [[nodiscard]] bool pushV128(V128 c) { return stack.append(Val(c)); } + [[nodiscard]] bool pushRef(ValType type, AnyRef ref) { + return stack.append(Val(type, ref)); + } + [[nodiscard]] bool pushFuncRef(HandleFuncRef ref) { + return stack.append(Val(RefType::func(), ref)); + } + +#if defined(ENABLE_WASM_EXTENDED_CONST) || defined(ENABLE_WASM_GC) + int32_t popI32() { + uint32_t result = stack.back().i32(); + stack.popBack(); + return int32_t(result); + } +#endif +#ifdef ENABLE_WASM_EXTENDED_CONST + int64_t popI64() { + uint64_t result = stack.back().i64(); + stack.popBack(); + return int64_t(result); + } +#endif + + bool evalGlobalGet(JSContext* cx, uint32_t index) { + RootedVal val(cx); + instance().constantGlobalGet(index, &val); + return stack.append(val); + } + bool evalI32Const(int32_t c) { return pushI32(c); } + bool evalI64Const(int64_t c) { return pushI64(c); } + bool evalF32Const(float c) { return pushF32(c); } + bool evalF64Const(double c) { return pushF64(c); } + bool evalV128Const(V128 c) { return pushV128(c); } + bool evalRefFunc(JSContext* cx, uint32_t funcIndex) { + RootedFuncRef func(cx, FuncRef::fromJSFunction(nullptr)); + if (!instance().constantRefFunc(funcIndex, &func)) { + return false; + } + return pushFuncRef(func); + } + bool evalRefNull(RefType type) { return pushRef(type, AnyRef::null()); } +#ifdef ENABLE_WASM_EXTENDED_CONST + bool evalI32Add() { + uint32_t b = popI32(); + uint32_t a = popI32(); + return pushI32(a + b); + } + bool evalI32Sub() { + uint32_t b = popI32(); + uint32_t a = popI32(); + return pushI32(a - b); + } + bool evalI32Mul() { + uint32_t b = popI32(); + uint32_t a = popI32(); + return pushI32(a * b); + } + bool evalI64Add() { + uint64_t b = popI64(); + uint64_t a = popI64(); + return pushI64(a + b); + } + bool evalI64Sub() { + uint64_t b = popI64(); + uint64_t a = popI64(); + return pushI64(a - b); + } + bool evalI64Mul() { + uint64_t b = popI64(); + uint64_t a = popI64(); + return pushI64(a * b); + } +#endif // ENABLE_WASM_EXTENDED_CONST +#ifdef ENABLE_WASM_GC + bool evalStructNew(JSContext* cx, uint32_t typeIndex) { + const TypeDef& typeDef = instance().metadata().types->type(typeIndex); + const StructType& structType = typeDef.structType(); + + Rooted structObj( + cx, instance().constantStructNewDefault(cx, typeIndex)); + if (!structObj) { + return false; + } + + uint32_t numFields = structType.fields_.length(); + for (uint32_t forwardIndex = 0; forwardIndex < numFields; forwardIndex++) { + uint32_t reverseIndex = numFields - forwardIndex - 1; + const Val& val = stack.back(); + structObj->storeVal(val, reverseIndex); + stack.popBack(); + } + + return pushRef(RefType::fromTypeDef(&typeDef, false), + AnyRef::fromJSObject(structObj)); + } + + bool evalStructNewDefault(JSContext* cx, uint32_t typeIndex) { + Rooted structObj( + cx, instance().constantStructNewDefault(cx, typeIndex)); + if (!structObj) { + return false; + } + + const TypeDef& typeDef = instance().metadata().types->type(typeIndex); + return pushRef(RefType::fromTypeDef(&typeDef, false), + AnyRef::fromJSObject(structObj)); + } + + bool evalArrayNew(JSContext* cx, uint32_t typeIndex) { + uint32_t numElements = popI32(); + Rooted arrayObj( + cx, instance().constantArrayNewDefault(cx, typeIndex, numElements)); + if (!arrayObj) { + return false; + } + + const Val& val = stack.back(); + arrayObj->fillVal(val, 0, numElements); + stack.popBack(); + + const TypeDef& typeDef = instance().metadata().types->type(typeIndex); + return pushRef(RefType::fromTypeDef(&typeDef, false), + AnyRef::fromJSObject(arrayObj)); + } + + bool evalArrayNewDefault(JSContext* cx, uint32_t typeIndex) { + uint32_t numElements = popI32(); + Rooted arrayObj( + cx, instance().constantArrayNewDefault(cx, typeIndex, numElements)); + if (!arrayObj) { + return false; + } + + const TypeDef& typeDef = instance().metadata().types->type(typeIndex); + return pushRef(RefType::fromTypeDef(&typeDef, false), + AnyRef::fromJSObject(arrayObj)); + } + + bool evalArrayNewFixed(JSContext* cx, uint32_t typeIndex, + uint32_t numElements) { + Rooted arrayObj( + cx, instance().constantArrayNewDefault(cx, typeIndex, numElements)); + if (!arrayObj) { + return false; + } + + for (uint32_t forwardIndex = 0; forwardIndex < numElements; + forwardIndex++) { + uint32_t reverseIndex = numElements - forwardIndex - 1; + const Val& val = stack.back(); + arrayObj->storeVal(val, reverseIndex); + stack.popBack(); + } + + const TypeDef& typeDef = instance().metadata().types->type(typeIndex); + return pushRef(RefType::fromTypeDef(&typeDef, false), + AnyRef::fromJSObject(arrayObj)); + } +#endif // ENABLE_WASM_GC +}; + +bool InitExprInterpreter::evaluate(JSContext* cx, Decoder& d) { +#define CHECK(c) \ + if (!(c)) return false; \ + break + + while (true) { + OpBytes op; + if (!d.readOp(&op)) { + return false; + } + + switch (op.b0) { + case uint16_t(Op::End): { + return true; + } + case uint16_t(Op::GlobalGet): { + uint32_t index; + if (!d.readGlobalIndex(&index)) { + return false; + } + CHECK(evalGlobalGet(cx, index)); + } + case uint16_t(Op::I32Const): { + int32_t c; + if (!d.readI32Const(&c)) { + return false; + } + CHECK(evalI32Const(c)); + } + case uint16_t(Op::I64Const): { + int64_t c; + if (!d.readI64Const(&c)) { + return false; + } + CHECK(evalI64Const(c)); + } + case uint16_t(Op::F32Const): { + float c; + if (!d.readF32Const(&c)) { + return false; + } + CHECK(evalF32Const(c)); + } + case uint16_t(Op::F64Const): { + double c; + if (!d.readF64Const(&c)) { + return false; + } + CHECK(evalF64Const(c)); + } +#ifdef ENABLE_WASM_SIMD + case uint16_t(Op::SimdPrefix): { + MOZ_RELEASE_ASSERT(op.b1 == uint32_t(SimdOp::V128Const)); + V128 c; + if (!d.readV128Const(&c)) { + return false; + } + CHECK(evalV128Const(c)); + } +#endif + case uint16_t(Op::RefFunc): { + uint32_t funcIndex; + if (!d.readFuncIndex(&funcIndex)) { + return false; + } + CHECK(evalRefFunc(cx, funcIndex)); + } + case uint16_t(Op::RefNull): { + RefType type; + if (!d.readRefNull(*types, features, &type)) { + return false; + } + CHECK(evalRefNull(type)); + } +#ifdef ENABLE_WASM_EXTENDED_CONST + case uint16_t(Op::I32Add): { + if (!d.readBinary()) { + return false; + } + CHECK(evalI32Add()); + } + case uint16_t(Op::I32Sub): { + if (!d.readBinary()) { + return false; + } + CHECK(evalI32Sub()); + } + case uint16_t(Op::I32Mul): { + if (!d.readBinary()) { + return false; + } + CHECK(evalI32Mul()); + } + case uint16_t(Op::I64Add): { + if (!d.readBinary()) { + return false; + } + CHECK(evalI64Add()); + } + case uint16_t(Op::I64Sub): { + if (!d.readBinary()) { + return false; + } + CHECK(evalI64Sub()); + } + case uint16_t(Op::I64Mul): { + if (!d.readBinary()) { + return false; + } + CHECK(evalI64Mul()); + } +#endif +#ifdef ENABLE_WASM_GC + case uint16_t(Op::GcPrefix): { + switch (op.b1) { + case uint32_t(GcOp::StructNew): { + uint32_t typeIndex; + if (!d.readTypeIndex(&typeIndex)) { + return false; + } + CHECK(evalStructNew(cx, typeIndex)); + } + case uint32_t(GcOp::StructNewDefault): { + uint32_t typeIndex; + if (!d.readTypeIndex(&typeIndex)) { + return false; + } + CHECK(evalStructNewDefault(cx, typeIndex)); + } + case uint32_t(GcOp::ArrayNew): { + uint32_t typeIndex; + if (!d.readTypeIndex(&typeIndex)) { + return false; + } + CHECK(evalArrayNew(cx, typeIndex)); + } + case uint32_t(GcOp::ArrayNewFixed): { + uint32_t typeIndex, len; + if (!d.readTypeIndex(&typeIndex)) { + return false; + } + if (!d.readVarU32(&len)) { + return false; + } + CHECK(evalArrayNewFixed(cx, typeIndex, len)); + } + case uint32_t(GcOp::ArrayNewDefault): { + uint32_t typeIndex; + if (!d.readTypeIndex(&typeIndex)) { + return false; + } + CHECK(evalArrayNewDefault(cx, typeIndex)); + } + default: { + MOZ_CRASH(); + } + } + break; + } +#endif + default: { + MOZ_CRASH(); + } + } + } + +#undef CHECK +} + +bool InitExpr::decodeAndValidate(Decoder& d, ModuleEnvironment* env, + ValType expected, + uint32_t maxInitializedGlobalsIndexPlus1, + InitExpr* expr) { + Maybe literal = Nothing(); + const uint8_t* exprStart = d.currentPosition(); + if (!ValidateInitExpr(d, env, expected, maxInitializedGlobalsIndexPlus1, + &literal)) { + return false; + } + const uint8_t* exprEnd = d.currentPosition(); + size_t exprSize = exprEnd - exprStart; + + MOZ_ASSERT(expr->kind_ == InitExprKind::None); + expr->type_ = expected; + + if (literal) { + expr->kind_ = InitExprKind::Literal; + expr->literal_ = *literal; + return true; + } + + expr->kind_ = InitExprKind::Variable; + return expr->bytecode_.reserve(exprSize) && + expr->bytecode_.append(exprStart, exprEnd); +} + +bool InitExpr::evaluate(JSContext* cx, Handle instanceObj, + MutableHandleVal result) const { + MOZ_ASSERT(kind_ != InitExprKind::None); + + if (isLiteral()) { + result.set(Val(literal())); + return true; + } + + UniqueChars error; + Decoder d(bytecode_.begin(), bytecode_.end(), 0, &error); + InitExprInterpreter interp(cx, instanceObj); + if (!interp.evaluate(cx, d)) { + // This expression should have been validated already. So we should only be + // able to OOM, which is reported by having no error message. + MOZ_RELEASE_ASSERT(!error); + return false; + } + + result.set(interp.result()); + return true; +} + +bool InitExpr::clone(const InitExpr& src) { + kind_ = src.kind_; + MOZ_ASSERT(bytecode_.empty()); + if (!bytecode_.appendAll(src.bytecode_)) { + return false; + } + literal_ = src.literal_; + type_ = src.type_; + return true; +} + +size_t InitExpr::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return bytecode_.sizeOfExcludingThis(mallocSizeOf); +} diff --git a/js/src/wasm/WasmInitExpr.h b/js/src/wasm/WasmInitExpr.h new file mode 100644 index 0000000000..385f1c6b66 --- /dev/null +++ b/js/src/wasm/WasmInitExpr.h @@ -0,0 +1,101 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_initexpr_h +#define wasm_initexpr_h + +#include "wasm/WasmConstants.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmValType.h" +#include "wasm/WasmValue.h" + +namespace js { +namespace wasm { + +class Decoder; +struct ModuleEnvironment; + +enum class InitExprKind { + None, + Literal, + Variable, +}; + +// A InitExpr describes a deferred initializer expression, used to initialize +// a global or a table element offset. Such expressions are created during +// decoding and actually executed on module instantiation. + +class InitExpr { + InitExprKind kind_; + // The bytecode for this constant expression if this is not a literal. + Bytes bytecode_; + // The value if this is a literal. + LitVal literal_; + // The value type of this constant expression in either case. + ValType type_; + + public: + InitExpr() : kind_(InitExprKind::None) {} + + explicit InitExpr(LitVal literal) + : kind_(InitExprKind::Literal), + literal_(literal), + type_(literal.type()) {} + + // Decode and validate a constant expression given at the current + // position of the decoder. Upon failure, the decoder contains the failure + // message or else the failure was an OOM. + static bool decodeAndValidate(Decoder& d, ModuleEnvironment* env, + ValType expected, + uint32_t maxInitializedGlobalsIndexPlus1, + InitExpr* expr); + + // Evaluate the constant expresssion with the given context. This may only + // fail due to an OOM, as all InitExpr's are required to have been validated. + bool evaluate(JSContext* cx, Handle instanceObj, + MutableHandleVal result) const; + + bool isLiteral() const { return kind_ == InitExprKind::Literal; } + + // Gets the result of this expression if it was determined to be a literal. + LitVal literal() const { + MOZ_ASSERT(isLiteral()); + return literal_; + } + + // Get the type of the resulting value of this expression. + ValType type() const { return type_; } + + // Allow moving, but not implicit copying + InitExpr(const InitExpr&) = delete; + InitExpr& operator=(const InitExpr&) = delete; + InitExpr(InitExpr&&) = default; + InitExpr& operator=(InitExpr&&) = default; + + // Allow explicit cloning + [[nodiscard]] bool clone(const InitExpr& src); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(InitExpr); +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_initexpr_h diff --git a/js/src/wasm/WasmInstance-inl.h b/js/src/wasm/WasmInstance-inl.h new file mode 100644 index 0000000000..519e718cf9 --- /dev/null +++ b/js/src/wasm/WasmInstance-inl.h @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef wasm_Instance_inl_h +#define wasm_Instance_inl_h + +#include "wasm/WasmInstance.h" + +#include "wasm/WasmCode.h" + +namespace js { +namespace wasm { + +const CodeTier& Instance::code(Tier t) const { return code_->codeTier(t); } + +uint8_t* Instance::codeBase(Tier t) const { return code_->segment(t).base(); } + +const MetadataTier& Instance::metadata(Tier t) const { + return code_->metadata(t); +} + +const Metadata& Instance::metadata() const { return code_->metadata(); } + +bool Instance::isAsmJS() const { return metadata().isAsmJS(); } + +} // namespace wasm +} // namespace js + +#endif // wasm_Instance_inl_h diff --git a/js/src/wasm/WasmInstance.cpp b/js/src/wasm/WasmInstance.cpp new file mode 100644 index 0000000000..557ed4e988 --- /dev/null +++ b/js/src/wasm/WasmInstance.cpp @@ -0,0 +1,2759 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmInstance-inl.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/DebugOnly.h" + +#include +#include + +#include "jsmath.h" + +#include "gc/Marking.h" +#include "jit/AtomicOperations.h" +#include "jit/Disassemble.h" +#include "jit/JitCommon.h" +#include "jit/JitRuntime.h" +#include "jit/Registers.h" +#include "js/ForOfIterator.h" +#include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_* +#include "js/Stack.h" // JS::NativeStackLimitMin +#include "util/StringBuffer.h" +#include "util/Text.h" +#include "vm/ArrayBufferObject.h" +#include "vm/BigIntType.h" +#include "vm/Compartment.h" +#include "vm/ErrorObject.h" +#include "vm/Interpreter.h" +#include "vm/Iteration.h" +#include "vm/JitActivation.h" +#include "vm/PlainObject.h" // js::PlainObject +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCode.h" +#include "wasm/WasmDebug.h" +#include "wasm/WasmDebugFrame.h" +#include "wasm/WasmGcObject.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmMemory.h" +#include "wasm/WasmModule.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValType.h" +#include "wasm/WasmValue.h" + +#include "gc/StoreBuffer-inl.h" +#include "vm/ArrayBufferObject-inl.h" +#include "vm/JSObject-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::BitwiseCast; +using mozilla::CheckedUint32; +using mozilla::DebugOnly; + +// Instance must be aligned at least as much as any of the integer, float, +// or SIMD values that we'd like to store in it. +static_assert(alignof(Instance) >= + std::max(sizeof(Registers::RegisterContent), + sizeof(FloatRegisters::RegisterContent))); + +// The globalArea must be aligned at least as much as an instance. This is +// guaranteed to be sufficient for all data types we care about, including +// SIMD values. See the above assertion. +static_assert(Instance::offsetOfData() % alignof(Instance) == 0); + +// We want the memory base to be the first field, and accessible with no +// offset. This incidentally is also an assertion that there is no superclass +// with fields. +static_assert(Instance::offsetOfMemoryBase() == 0); + +// We want instance fields that are commonly accessed by the JIT to have +// compact encodings. A limit of less than 128 bytes is chosen to fit within +// the signed 8-bit mod r/m x86 encoding. +static_assert(Instance::offsetOfLastCommonJitField() < 128); + +////////////////////////////////////////////////////////////////////////////// +// +// Functions and invocation. + +TypeDefInstanceData* Instance::typeDefInstanceData(uint32_t typeIndex) const { + TypeDefInstanceData* instanceData = + (TypeDefInstanceData*)(data() + metadata().typeDefsOffsetStart); + return &instanceData[typeIndex]; +} + +const void* Instance::addressOfGlobalCell(const GlobalDesc& global) const { + const void* cell = data() + global.offset(); + // Indirect globals store a pointer to their cell in the instance global + // data. Dereference it to find the real cell. + if (global.isIndirect()) { + cell = *(const void**)cell; + } + return cell; +} + +FuncImportInstanceData& Instance::funcImportInstanceData(const FuncImport& fi) { + return *(FuncImportInstanceData*)(data() + fi.instanceOffset()); +} + +TableInstanceData& Instance::tableInstanceData(uint32_t tableIndex) const { + TableInstanceData* instanceData = + (TableInstanceData*)(data() + metadata().tablesOffsetStart); + return instanceData[tableIndex]; +} + +TagInstanceData& Instance::tagInstanceData(uint32_t tagIndex) const { + TagInstanceData* instanceData = + (TagInstanceData*)(data() + metadata().tagsOffsetStart); + return instanceData[tagIndex]; +} + +static bool UnpackResults(JSContext* cx, const ValTypeVector& resultTypes, + const Maybe stackResultsArea, uint64_t* argv, + MutableHandleValue rval) { + if (!stackResultsArea) { + MOZ_ASSERT(resultTypes.length() <= 1); + // Result is either one scalar value to unpack to a wasm value, or + // an ignored value for a zero-valued function. + if (resultTypes.length() == 1) { + return ToWebAssemblyValue(cx, rval, resultTypes[0], argv, true); + } + return true; + } + + MOZ_ASSERT(stackResultsArea.isSome()); + Rooted array(cx); + if (!IterableToArray(cx, rval, &array)) { + return false; + } + + if (resultTypes.length() != array->length()) { + UniqueChars expected(JS_smprintf("%zu", resultTypes.length())); + UniqueChars got(JS_smprintf("%u", array->length())); + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_WRONG_NUMBER_OF_VALUES, expected.get(), + got.get()); + return false; + } + + DebugOnly previousOffset = ~(uint64_t)0; + + ABIResultIter iter(ResultType::Vector(resultTypes)); + // The values are converted in the order they are pushed on the + // abstract WebAssembly stack; switch to iterate in push order. + while (!iter.done()) { + iter.next(); + } + DebugOnly seenRegisterResult = false; + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + MOZ_ASSERT(!seenRegisterResult); + // Use rval as a scratch area to hold the extracted result. + rval.set(array->getDenseElement(iter.index())); + if (result.inRegister()) { + // Currently, if a function type has results, there can be only + // one register result. If there is only one result, it is + // returned as a scalar and not an iterable, so we don't get here. + // If there are multiple results, we extract the register result + // and set `argv[0]` set to the extracted result, to be returned by + // register in the stub. The register result follows any stack + // results, so this preserves conversion order. + if (!ToWebAssemblyValue(cx, rval, result.type(), argv, true)) { + return false; + } + seenRegisterResult = true; + continue; + } + uint32_t result_size = result.size(); + MOZ_ASSERT(result_size == 4 || result_size == 8); +#ifdef DEBUG + if (previousOffset == ~(uint64_t)0) { + previousOffset = (uint64_t)result.stackOffset(); + } else { + MOZ_ASSERT(previousOffset - (uint64_t)result_size == + (uint64_t)result.stackOffset()); + previousOffset -= (uint64_t)result_size; + } +#endif + char* loc = stackResultsArea.value() + result.stackOffset(); + if (!ToWebAssemblyValue(cx, rval, result.type(), loc, result_size == 8)) { + return false; + } + } + + return true; +} + +bool Instance::callImport(JSContext* cx, uint32_t funcImportIndex, + unsigned argc, uint64_t* argv) { + AssertRealmUnchanged aru(cx); + + Tier tier = code().bestTier(); + + const FuncImport& fi = metadata(tier).funcImports[funcImportIndex]; + const FuncType& funcType = metadata().getFuncImportType(fi); + + ArgTypeVector argTypes(funcType); + InvokeArgs args(cx); + if (!args.init(cx, argTypes.lengthWithoutStackResults())) { + return false; + } + + if (funcType.hasUnexposableArgOrRet()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + + MOZ_ASSERT(argTypes.lengthWithStackResults() == argc); + Maybe stackResultPointer; + size_t lastBoxIndexPlusOne = 0; + { + JS::AutoAssertNoGC nogc; + for (size_t i = 0; i < argc; i++) { + const void* rawArgLoc = &argv[i]; + if (argTypes.isSyntheticStackResultPointerArg(i)) { + stackResultPointer = Some(*(char**)rawArgLoc); + continue; + } + size_t naturalIndex = argTypes.naturalIndex(i); + ValType type = funcType.args()[naturalIndex]; + // Avoid boxes creation not to trigger GC. + if (ToJSValueMayGC(type)) { + lastBoxIndexPlusOne = i + 1; + continue; + } + MutableHandleValue argValue = args[naturalIndex]; + if (!ToJSValue(cx, rawArgLoc, type, argValue)) { + return false; + } + } + } + + // Visit arguments that need to perform allocation in a second loop + // after the rest of arguments are converted. + for (size_t i = 0; i < lastBoxIndexPlusOne; i++) { + if (argTypes.isSyntheticStackResultPointerArg(i)) { + continue; + } + const void* rawArgLoc = &argv[i]; + size_t naturalIndex = argTypes.naturalIndex(i); + ValType type = funcType.args()[naturalIndex]; + if (!ToJSValueMayGC(type)) { + continue; + } + MOZ_ASSERT(!type.isRefRepr()); + // The conversions are safe here because source values are not references + // and will not be moved. + MutableHandleValue argValue = args[naturalIndex]; + if (!ToJSValue(cx, rawArgLoc, type, argValue)) { + return false; + } + } + + FuncImportInstanceData& import = funcImportInstanceData(fi); + Rooted importCallable(cx, import.callable); + MOZ_ASSERT(cx->realm() == importCallable->nonCCWRealm()); + + RootedValue fval(cx, ObjectValue(*importCallable)); + RootedValue thisv(cx, UndefinedValue()); + RootedValue rval(cx); + if (!Call(cx, fval, thisv, args, &rval)) { + return false; + } + + if (!UnpackResults(cx, funcType.results(), stackResultPointer, argv, &rval)) { + return false; + } + + if (!JitOptions.enableWasmJitExit) { + return true; + } + + // The import may already have become optimized. + for (auto t : code().tiers()) { + void* jitExitCode = codeBase(t) + fi.jitExitCodeOffset(); + if (import.code == jitExitCode) { + return true; + } + } + + void* jitExitCode = codeBase(tier) + fi.jitExitCodeOffset(); + + if (!importCallable->is()) { + return true; + } + + // Test if the function is JIT compiled. + if (!importCallable->as().hasBytecode()) { + return true; + } + + JSScript* script = importCallable->as().nonLazyScript(); + if (!script->hasJitScript()) { + return true; + } + + // Skip if the function does not have a signature that allows for a JIT exit. + if (!funcType.canHaveJitExit()) { + return true; + } + + // Let's optimize it! + + import.code = jitExitCode; + return true; +} + +/* static */ int32_t /* 0 to signal trap; 1 to signal OK */ +Instance::callImport_general(Instance* instance, int32_t funcImportIndex, + int32_t argc, uint64_t* argv) { + JSContext* cx = instance->cx(); + return instance->callImport(cx, funcImportIndex, argc, argv); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Atomic operations and shared memory. + +template +static int32_t PerformWait(Instance* instance, PtrT byteOffset, ValT value, + int64_t timeout_ns) { + JSContext* cx = instance->cx(); + + if (!instance->memory()->isShared()) { + ReportTrapError(cx, JSMSG_WASM_NONSHARED_WAIT); + return -1; + } + + if (byteOffset & (sizeof(ValT) - 1)) { + ReportTrapError(cx, JSMSG_WASM_UNALIGNED_ACCESS); + return -1; + } + + if (byteOffset + sizeof(ValT) > instance->memory()->volatileMemoryLength()) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + mozilla::Maybe timeout; + if (timeout_ns >= 0) { + timeout = mozilla::Some( + mozilla::TimeDuration::FromMicroseconds(double(timeout_ns) / 1000)); + } + + MOZ_ASSERT(byteOffset <= SIZE_MAX, "Bounds check is broken"); + switch (atomics_wait_impl(cx, instance->sharedMemoryBuffer(), + size_t(byteOffset), value, timeout)) { + case FutexThread::WaitResult::OK: + return 0; + case FutexThread::WaitResult::NotEqual: + return 1; + case FutexThread::WaitResult::TimedOut: + return 2; + case FutexThread::WaitResult::Error: + return -1; + default: + MOZ_CRASH(); + } +} + +/* static */ int32_t Instance::wait_i32_m32(Instance* instance, + uint32_t byteOffset, int32_t value, + int64_t timeout_ns) { + MOZ_ASSERT(SASigWaitI32M32.failureMode == FailureMode::FailOnNegI32); + return PerformWait(instance, byteOffset, value, timeout_ns); +} + +/* static */ int32_t Instance::wait_i32_m64(Instance* instance, + uint64_t byteOffset, int32_t value, + int64_t timeout_ns) { + MOZ_ASSERT(SASigWaitI32M64.failureMode == FailureMode::FailOnNegI32); + return PerformWait(instance, byteOffset, value, timeout_ns); +} + +/* static */ int32_t Instance::wait_i64_m32(Instance* instance, + uint32_t byteOffset, int64_t value, + int64_t timeout_ns) { + MOZ_ASSERT(SASigWaitI64M32.failureMode == FailureMode::FailOnNegI32); + return PerformWait(instance, byteOffset, value, timeout_ns); +} + +/* static */ int32_t Instance::wait_i64_m64(Instance* instance, + uint64_t byteOffset, int64_t value, + int64_t timeout_ns) { + MOZ_ASSERT(SASigWaitI64M64.failureMode == FailureMode::FailOnNegI32); + return PerformWait(instance, byteOffset, value, timeout_ns); +} + +template +static int32_t PerformWake(Instance* instance, PtrT byteOffset, int32_t count) { + JSContext* cx = instance->cx(); + + // The alignment guard is not in the wasm spec as of 2017-11-02, but is + // considered likely to appear, as 4-byte alignment is required for WAKE by + // the spec's validation algorithm. + + if (byteOffset & 3) { + ReportTrapError(cx, JSMSG_WASM_UNALIGNED_ACCESS); + return -1; + } + + if (byteOffset >= instance->memory()->volatileMemoryLength()) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + if (!instance->memory()->isShared()) { + return 0; + } + + MOZ_ASSERT(byteOffset <= SIZE_MAX, "Bounds check is broken"); + int64_t woken = atomics_notify_impl(instance->sharedMemoryBuffer(), + size_t(byteOffset), int64_t(count)); + + if (woken > INT32_MAX) { + ReportTrapError(cx, JSMSG_WASM_WAKE_OVERFLOW); + return -1; + } + + return int32_t(woken); +} + +/* static */ int32_t Instance::wake_m32(Instance* instance, uint32_t byteOffset, + int32_t count) { + MOZ_ASSERT(SASigWakeM32.failureMode == FailureMode::FailOnNegI32); + return PerformWake(instance, byteOffset, count); +} + +/* static */ int32_t Instance::wake_m64(Instance* instance, uint64_t byteOffset, + int32_t count) { + MOZ_ASSERT(SASigWakeM32.failureMode == FailureMode::FailOnNegI32); + return PerformWake(instance, byteOffset, count); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk memory operations. + +/* static */ uint32_t Instance::memoryGrow_m32(Instance* instance, + uint32_t delta) { + MOZ_ASSERT(SASigMemoryGrowM32.failureMode == FailureMode::Infallible); + MOZ_ASSERT(!instance->isAsmJS()); + + JSContext* cx = instance->cx(); + Rooted memory(cx, instance->memory_); + + // It is safe to cast to uint32_t, as all limits have been checked inside + // grow() and will not have been exceeded for a 32-bit memory. + uint32_t ret = uint32_t(WasmMemoryObject::grow(memory, uint64_t(delta), cx)); + + // If there has been a moving grow, this Instance should have been notified. + MOZ_RELEASE_ASSERT(instance->memoryBase_ == + instance->memory_->buffer().dataPointerEither()); + + return ret; +} + +/* static */ uint64_t Instance::memoryGrow_m64(Instance* instance, + uint64_t delta) { + MOZ_ASSERT(SASigMemoryGrowM64.failureMode == FailureMode::Infallible); + MOZ_ASSERT(!instance->isAsmJS()); + + JSContext* cx = instance->cx(); + Rooted memory(cx, instance->memory_); + + uint64_t ret = WasmMemoryObject::grow(memory, delta, cx); + + // If there has been a moving grow, this Instance should have been notified. + MOZ_RELEASE_ASSERT(instance->memoryBase_ == + instance->memory_->buffer().dataPointerEither()); + + return ret; +} + +/* static */ uint32_t Instance::memorySize_m32(Instance* instance) { + MOZ_ASSERT(SASigMemorySizeM32.failureMode == FailureMode::Infallible); + + // This invariant must hold when running Wasm code. Assert it here so we can + // write tests for cross-realm calls. + DebugOnly cx = instance->cx(); + MOZ_ASSERT(cx->realm() == instance->realm()); + + Pages pages = instance->memory()->volatilePages(); +#ifdef JS_64BIT + // Ensure that the memory size is no more than 4GiB. + MOZ_ASSERT(pages <= Pages(MaxMemory32LimitField)); +#endif + return uint32_t(pages.value()); +} + +/* static */ uint64_t Instance::memorySize_m64(Instance* instance) { + MOZ_ASSERT(SASigMemorySizeM64.failureMode == FailureMode::Infallible); + + // This invariant must hold when running Wasm code. Assert it here so we can + // write tests for cross-realm calls. + DebugOnly cx = instance->cx(); + MOZ_ASSERT(cx->realm() == instance->realm()); + + Pages pages = instance->memory()->volatilePages(); +#ifdef JS_64BIT + MOZ_ASSERT(pages <= Pages(MaxMemory64LimitField)); +#endif + return pages.value(); +} + +template +inline int32_t WasmMemoryCopy(JSContext* cx, T memBase, size_t memLen, + I dstByteOffset, I srcByteOffset, I len, + F memMove) { + if (!MemoryBoundsCheck(dstByteOffset, len, memLen) || + !MemoryBoundsCheck(srcByteOffset, len, memLen)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + memMove(memBase + uintptr_t(dstByteOffset), + memBase + uintptr_t(srcByteOffset), size_t(len)); + return 0; +} + +template +inline int32_t MemoryCopy(JSContext* cx, I dstByteOffset, I srcByteOffset, + I len, uint8_t* memBase) { + const WasmArrayRawBuffer* rawBuf = WasmArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->byteLength(); + return WasmMemoryCopy(cx, memBase, memLen, dstByteOffset, srcByteOffset, len, + memmove); +} + +template +inline int32_t MemoryCopyShared(JSContext* cx, I dstByteOffset, I srcByteOffset, + I len, uint8_t* memBase) { + using RacyMemMove = + void (*)(SharedMem, SharedMem, size_t); + + const WasmSharedArrayRawBuffer* rawBuf = + WasmSharedArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->volatileByteLength(); + + return WasmMemoryCopy, RacyMemMove>( + cx, SharedMem::shared(memBase), memLen, dstByteOffset, + srcByteOffset, len, AtomicOperations::memmoveSafeWhenRacy); +} + +/* static */ int32_t Instance::memCopy_m32(Instance* instance, + uint32_t dstByteOffset, + uint32_t srcByteOffset, uint32_t len, + uint8_t* memBase) { + MOZ_ASSERT(SASigMemCopyM32.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryCopy(cx, dstByteOffset, srcByteOffset, len, memBase); +} + +/* static */ int32_t Instance::memCopyShared_m32(Instance* instance, + uint32_t dstByteOffset, + uint32_t srcByteOffset, + uint32_t len, + uint8_t* memBase) { + MOZ_ASSERT(SASigMemCopySharedM32.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryCopyShared(cx, dstByteOffset, srcByteOffset, len, memBase); +} + +/* static */ int32_t Instance::memCopy_m64(Instance* instance, + uint64_t dstByteOffset, + uint64_t srcByteOffset, uint64_t len, + uint8_t* memBase) { + MOZ_ASSERT(SASigMemCopyM64.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryCopy(cx, dstByteOffset, srcByteOffset, len, memBase); +} + +/* static */ int32_t Instance::memCopyShared_m64(Instance* instance, + uint64_t dstByteOffset, + uint64_t srcByteOffset, + uint64_t len, + uint8_t* memBase) { + MOZ_ASSERT(SASigMemCopySharedM64.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryCopyShared(cx, dstByteOffset, srcByteOffset, len, memBase); +} + +template +inline int32_t WasmMemoryFill(JSContext* cx, T memBase, size_t memLen, + I byteOffset, uint32_t value, I len, F memSet) { + if (!MemoryBoundsCheck(byteOffset, len, memLen)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + // The required write direction is upward, but that is not currently + // observable as there are no fences nor any read/write protect operation. + memSet(memBase + uintptr_t(byteOffset), int(value), size_t(len)); + return 0; +} + +template +inline int32_t MemoryFill(JSContext* cx, I byteOffset, uint32_t value, I len, + uint8_t* memBase) { + const WasmArrayRawBuffer* rawBuf = WasmArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->byteLength(); + return WasmMemoryFill(cx, memBase, memLen, byteOffset, value, len, memset); +} + +template +inline int32_t MemoryFillShared(JSContext* cx, I byteOffset, uint32_t value, + I len, uint8_t* memBase) { + const WasmSharedArrayRawBuffer* rawBuf = + WasmSharedArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->volatileByteLength(); + return WasmMemoryFill(cx, SharedMem::shared(memBase), memLen, + byteOffset, value, len, + AtomicOperations::memsetSafeWhenRacy); +} + +/* static */ int32_t Instance::memFill_m32(Instance* instance, + uint32_t byteOffset, uint32_t value, + uint32_t len, uint8_t* memBase) { + MOZ_ASSERT(SASigMemFillM32.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryFill(cx, byteOffset, value, len, memBase); +} + +/* static */ int32_t Instance::memFillShared_m32(Instance* instance, + uint32_t byteOffset, + uint32_t value, uint32_t len, + uint8_t* memBase) { + MOZ_ASSERT(SASigMemFillSharedM32.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryFillShared(cx, byteOffset, value, len, memBase); +} + +/* static */ int32_t Instance::memFill_m64(Instance* instance, + uint64_t byteOffset, uint32_t value, + uint64_t len, uint8_t* memBase) { + MOZ_ASSERT(SASigMemFillM64.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryFill(cx, byteOffset, value, len, memBase); +} + +/* static */ int32_t Instance::memFillShared_m64(Instance* instance, + uint64_t byteOffset, + uint32_t value, uint64_t len, + uint8_t* memBase) { + MOZ_ASSERT(SASigMemFillSharedM64.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + return MemoryFillShared(cx, byteOffset, value, len, memBase); +} + +static bool BoundsCheckInit(uint32_t dstOffset, uint32_t srcOffset, + uint32_t len, size_t memLen, uint32_t segLen) { + uint64_t dstOffsetLimit = uint64_t(dstOffset) + uint64_t(len); + uint64_t srcOffsetLimit = uint64_t(srcOffset) + uint64_t(len); + + return dstOffsetLimit > memLen || srcOffsetLimit > segLen; +} + +static bool BoundsCheckInit(uint64_t dstOffset, uint32_t srcOffset, + uint32_t len, size_t memLen, uint32_t segLen) { + uint64_t dstOffsetLimit = dstOffset + uint64_t(len); + uint64_t srcOffsetLimit = uint64_t(srcOffset) + uint64_t(len); + + return dstOffsetLimit < dstOffset || dstOffsetLimit > memLen || + srcOffsetLimit > segLen; +} + +template +static int32_t MemoryInit(JSContext* cx, Instance* instance, I dstOffset, + uint32_t srcOffset, uint32_t len, + const DataSegment* maybeSeg) { + if (!maybeSeg) { + if (len == 0 && srcOffset == 0) { + return 0; + } + + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + const DataSegment& seg = *maybeSeg; + MOZ_RELEASE_ASSERT(!seg.active()); + + const uint32_t segLen = seg.bytes.length(); + + WasmMemoryObject* mem = instance->memory(); + const size_t memLen = mem->volatileMemoryLength(); + + // We are proposing to copy + // + // seg.bytes.begin()[ srcOffset .. srcOffset + len - 1 ] + // to + // memoryBase[ dstOffset .. dstOffset + len - 1 ] + + if (BoundsCheckInit(dstOffset, srcOffset, len, memLen, segLen)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + // The required read/write direction is upward, but that is not currently + // observable as there are no fences nor any read/write protect operation. + SharedMem dataPtr = mem->buffer().dataPointerEither(); + if (mem->isShared()) { + AtomicOperations::memcpySafeWhenRacy( + dataPtr + uintptr_t(dstOffset), (uint8_t*)seg.bytes.begin() + srcOffset, + len); + } else { + uint8_t* rawBuf = dataPtr.unwrap(/*Unshared*/); + memcpy(rawBuf + uintptr_t(dstOffset), + (const char*)seg.bytes.begin() + srcOffset, len); + } + return 0; +} + +/* static */ int32_t Instance::memInit_m32(Instance* instance, + uint32_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t segIndex) { + MOZ_ASSERT(SASigMemInitM32.failureMode == FailureMode::FailOnNegI32); + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveDataSegments_.length(), + "ensured by validation"); + + JSContext* cx = instance->cx(); + return MemoryInit(cx, instance, dstOffset, srcOffset, len, + instance->passiveDataSegments_[segIndex]); +} + +/* static */ int32_t Instance::memInit_m64(Instance* instance, + uint64_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t segIndex) { + MOZ_ASSERT(SASigMemInitM64.failureMode == FailureMode::FailOnNegI32); + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveDataSegments_.length(), + "ensured by validation"); + + JSContext* cx = instance->cx(); + return MemoryInit(cx, instance, dstOffset, srcOffset, len, + instance->passiveDataSegments_[segIndex]); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Bulk table operations. + +/* static */ int32_t Instance::tableCopy(Instance* instance, uint32_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t dstTableIndex, + uint32_t srcTableIndex) { + MOZ_ASSERT(SASigTableCopy.failureMode == FailureMode::FailOnNegI32); + + JSContext* cx = instance->cx(); + const SharedTable& srcTable = instance->tables()[srcTableIndex]; + uint32_t srcTableLen = srcTable->length(); + + const SharedTable& dstTable = instance->tables()[dstTableIndex]; + uint32_t dstTableLen = dstTable->length(); + + // Bounds check and deal with arithmetic overflow. + uint64_t dstOffsetLimit = uint64_t(dstOffset) + len; + uint64_t srcOffsetLimit = uint64_t(srcOffset) + len; + + if (dstOffsetLimit > dstTableLen || srcOffsetLimit > srcTableLen) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + bool isOOM = false; + + if (&srcTable == &dstTable && dstOffset > srcOffset) { + for (uint32_t i = len; i > 0; i--) { + if (!dstTable->copy(cx, *srcTable, dstOffset + (i - 1), + srcOffset + (i - 1))) { + isOOM = true; + break; + } + } + } else if (&srcTable == &dstTable && dstOffset == srcOffset) { + // No-op + } else { + for (uint32_t i = 0; i < len; i++) { + if (!dstTable->copy(cx, *srcTable, dstOffset + i, srcOffset + i)) { + isOOM = true; + break; + } + } + } + + if (isOOM) { + return -1; + } + return 0; +} + +bool Instance::initElems(uint32_t tableIndex, const ElemSegment& seg, + uint32_t dstOffset, uint32_t srcOffset, uint32_t len) { + Table& table = *tables_[tableIndex]; + MOZ_ASSERT(dstOffset <= table.length()); + MOZ_ASSERT(len <= table.length() - dstOffset); + + Tier tier = code().bestTier(); + const MetadataTier& metadataTier = metadata(tier); + const FuncImportVector& funcImports = metadataTier.funcImports; + const CodeRangeVector& codeRanges = metadataTier.codeRanges; + const Uint32Vector& funcToCodeRange = metadataTier.funcToCodeRange; + const Uint32Vector& elemFuncIndices = seg.elemFuncIndices; + MOZ_ASSERT(srcOffset <= elemFuncIndices.length()); + MOZ_ASSERT(len <= elemFuncIndices.length() - srcOffset); + + uint8_t* codeBaseTier = codeBase(tier); + for (uint32_t i = 0; i < len; i++) { + uint32_t funcIndex = elemFuncIndices[srcOffset + i]; + if (funcIndex == NullFuncIndex) { + table.setNull(dstOffset + i); + } else if (!table.isFunction()) { + // Note, fnref must be rooted if we do anything more than just store it. + void* fnref = Instance::refFunc(this, funcIndex); + if (fnref == AnyRef::invalid().forCompiledCode()) { + return false; // OOM, which has already been reported. + } + table.fillAnyRef(dstOffset + i, 1, AnyRef::fromCompiledCode(fnref)); + } else { + if (funcIndex < metadataTier.funcImports.length()) { + FuncImportInstanceData& import = + funcImportInstanceData(funcImports[funcIndex]); + MOZ_ASSERT(import.callable->isCallable()); + if (import.callable->is()) { + JSFunction* fun = &import.callable->as(); + if (IsWasmExportedFunction(fun)) { + // This element is a wasm function imported from another + // instance. To preserve the === function identity required by + // the JS embedding spec, we must set the element to the + // imported function's underlying CodeRange.funcCheckedCallEntry and + // Instance so that future Table.get()s produce the same + // function object as was imported. + WasmInstanceObject* calleeInstanceObj = + ExportedFunctionToInstanceObject(fun); + Instance& calleeInstance = calleeInstanceObj->instance(); + Tier calleeTier = calleeInstance.code().bestTier(); + const CodeRange& calleeCodeRange = + calleeInstanceObj->getExportedFunctionCodeRange(fun, + calleeTier); + void* code = calleeInstance.codeBase(calleeTier) + + calleeCodeRange.funcCheckedCallEntry(); + table.setFuncRef(dstOffset + i, code, &calleeInstance); + continue; + } + } + } + void* code = + codeBaseTier + + codeRanges[funcToCodeRange[funcIndex]].funcCheckedCallEntry(); + table.setFuncRef(dstOffset + i, code, this); + } + } + return true; +} + +/* static */ int32_t Instance::tableInit(Instance* instance, uint32_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t segIndex, + uint32_t tableIndex) { + MOZ_ASSERT(SASigTableInit.failureMode == FailureMode::FailOnNegI32); + + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveElemSegments_.length(), + "ensured by validation"); + + JSContext* cx = instance->cx(); + if (!instance->passiveElemSegments_[segIndex]) { + if (len == 0 && srcOffset == 0) { + return 0; + } + + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + const ElemSegment& seg = *instance->passiveElemSegments_[segIndex]; + MOZ_RELEASE_ASSERT(!seg.active()); + const uint32_t segLen = seg.length(); + + const Table& table = *instance->tables()[tableIndex]; + const uint32_t tableLen = table.length(); + + // We are proposing to copy + // + // seg[ srcOffset .. srcOffset + len - 1 ] + // to + // tableBase[ dstOffset .. dstOffset + len - 1 ] + + // Bounds check and deal with arithmetic overflow. + uint64_t dstOffsetLimit = uint64_t(dstOffset) + uint64_t(len); + uint64_t srcOffsetLimit = uint64_t(srcOffset) + uint64_t(len); + + if (dstOffsetLimit > tableLen || srcOffsetLimit > segLen) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + if (!instance->initElems(tableIndex, seg, dstOffset, srcOffset, len)) { + return -1; // OOM, which has already been reported. + } + + return 0; +} + +/* static */ int32_t Instance::tableFill(Instance* instance, uint32_t start, + void* value, uint32_t len, + uint32_t tableIndex) { + MOZ_ASSERT(SASigTableFill.failureMode == FailureMode::FailOnNegI32); + + JSContext* cx = instance->cx(); + Table& table = *instance->tables()[tableIndex]; + + // Bounds check and deal with arithmetic overflow. + uint64_t offsetLimit = uint64_t(start) + uint64_t(len); + + if (offsetLimit > table.length()) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + switch (table.repr()) { + case TableRepr::Ref: + table.fillAnyRef(start, len, AnyRef::fromCompiledCode(value)); + break; + case TableRepr::Func: + MOZ_RELEASE_ASSERT(!table.isAsmJS()); + table.fillFuncRef(start, len, FuncRef::fromCompiledCode(value), cx); + break; + } + + return 0; +} + +template +static bool WasmDiscardCheck(Instance* instance, I byteOffset, I byteLen, + size_t memLen, bool shared) { + JSContext* cx = instance->cx(); + + if (byteOffset % wasm::PageSize != 0 || byteLen % wasm::PageSize != 0) { + ReportTrapError(cx, JSMSG_WASM_UNALIGNED_ACCESS); + return false; + } + + if (!MemoryBoundsCheck(byteOffset, byteLen, memLen)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return false; + } + + return true; +} + +template +static int32_t MemDiscardNotShared(Instance* instance, I byteOffset, I byteLen, + uint8_t* memBase) { + WasmArrayRawBuffer* rawBuf = WasmArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->byteLength(); + + if (!WasmDiscardCheck(instance, byteOffset, byteLen, memLen, false)) { + return -1; + } + rawBuf->discard(byteOffset, byteLen); + + return 0; +} + +template +static int32_t MemDiscardShared(Instance* instance, I byteOffset, I byteLen, + uint8_t* memBase) { + WasmSharedArrayRawBuffer* rawBuf = + WasmSharedArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->volatileByteLength(); + + if (!WasmDiscardCheck(instance, byteOffset, byteLen, memLen, true)) { + return -1; + } + rawBuf->discard(byteOffset, byteLen); + + return 0; +} + +/* static */ int32_t Instance::memDiscard_m32(Instance* instance, + uint32_t byteOffset, + uint32_t byteLen, + uint8_t* memBase) { + return MemDiscardNotShared(instance, byteOffset, byteLen, memBase); +} + +/* static */ int32_t Instance::memDiscard_m64(Instance* instance, + uint64_t byteOffset, + uint64_t byteLen, + uint8_t* memBase) { + return MemDiscardNotShared(instance, byteOffset, byteLen, memBase); +} + +/* static */ int32_t Instance::memDiscardShared_m32(Instance* instance, + uint32_t byteOffset, + uint32_t byteLen, + uint8_t* memBase) { + return MemDiscardShared(instance, byteOffset, byteLen, memBase); +} + +/* static */ int32_t Instance::memDiscardShared_m64(Instance* instance, + uint64_t byteOffset, + uint64_t byteLen, + uint8_t* memBase) { + return MemDiscardShared(instance, byteOffset, byteLen, memBase); +} + +/* static */ void* Instance::tableGet(Instance* instance, uint32_t index, + uint32_t tableIndex) { + MOZ_ASSERT(SASigTableGet.failureMode == FailureMode::FailOnInvalidRef); + + JSContext* cx = instance->cx(); + const Table& table = *instance->tables()[tableIndex]; + if (index >= table.length()) { + ReportTrapError(cx, JSMSG_WASM_TABLE_OUT_OF_BOUNDS); + return AnyRef::invalid().forCompiledCode(); + } + + switch (table.repr()) { + case TableRepr::Ref: + return table.getAnyRef(index).forCompiledCode(); + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(!table.isAsmJS()); + RootedFunction fun(cx); + if (!table.getFuncRef(cx, index, &fun)) { + return AnyRef::invalid().forCompiledCode(); + } + return FuncRef::fromJSFunction(fun).forCompiledCode(); + } + } + MOZ_CRASH("Should not happen"); +} + +/* static */ uint32_t Instance::tableGrow(Instance* instance, void* initValue, + uint32_t delta, uint32_t tableIndex) { + MOZ_ASSERT(SASigTableGrow.failureMode == FailureMode::Infallible); + + JSContext* cx = instance->cx(); + RootedAnyRef ref(cx, AnyRef::fromCompiledCode(initValue)); + Table& table = *instance->tables()[tableIndex]; + + uint32_t oldSize = table.grow(delta); + + if (oldSize != uint32_t(-1) && initValue != nullptr) { + table.fillUninitialized(oldSize, delta, ref, cx); + } + +#ifdef DEBUG + if (!table.elemType().isNullable()) { + table.assertRangeNotNull(oldSize, delta); + } +#endif // DEBUG + return oldSize; +} + +/* static */ int32_t Instance::tableSet(Instance* instance, uint32_t index, + void* value, uint32_t tableIndex) { + MOZ_ASSERT(SASigTableSet.failureMode == FailureMode::FailOnNegI32); + + JSContext* cx = instance->cx(); + Table& table = *instance->tables()[tableIndex]; + + if (index >= table.length()) { + ReportTrapError(cx, JSMSG_WASM_TABLE_OUT_OF_BOUNDS); + return -1; + } + + switch (table.repr()) { + case TableRepr::Ref: + table.fillAnyRef(index, 1, AnyRef::fromCompiledCode(value)); + break; + case TableRepr::Func: + MOZ_RELEASE_ASSERT(!table.isAsmJS()); + table.fillFuncRef(index, 1, FuncRef::fromCompiledCode(value), cx); + break; + } + + return 0; +} + +/* static */ uint32_t Instance::tableSize(Instance* instance, + uint32_t tableIndex) { + MOZ_ASSERT(SASigTableSize.failureMode == FailureMode::Infallible); + Table& table = *instance->tables()[tableIndex]; + return table.length(); +} + +/* static */ void* Instance::refFunc(Instance* instance, uint32_t funcIndex) { + MOZ_ASSERT(SASigRefFunc.failureMode == FailureMode::FailOnInvalidRef); + JSContext* cx = instance->cx(); + + Tier tier = instance->code().bestTier(); + const MetadataTier& metadataTier = instance->metadata(tier); + const FuncImportVector& funcImports = metadataTier.funcImports; + + // If this is an import, we need to recover the original function to maintain + // reference equality between a re-exported function and 'ref.func'. The + // identity of the imported function object is stable across tiers, which is + // what we want. + // + // Use the imported function only if it is an exported function, otherwise + // fall through to get a (possibly new) exported function. + if (funcIndex < funcImports.length()) { + FuncImportInstanceData& import = + instance->funcImportInstanceData(funcImports[funcIndex]); + if (import.callable->is()) { + JSFunction* fun = &import.callable->as(); + if (IsWasmExportedFunction(fun)) { + return FuncRef::fromJSFunction(fun).forCompiledCode(); + } + } + } + + RootedFunction fun(cx); + Rooted instanceObj(cx, instance->object()); + if (!WasmInstanceObject::getExportedFunction(cx, instanceObj, funcIndex, + &fun)) { + // Validation ensures that we always have a valid funcIndex, so we must + // have OOM'ed + ReportOutOfMemory(cx); + return AnyRef::invalid().forCompiledCode(); + } + + return FuncRef::fromJSFunction(fun).forCompiledCode(); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Segment management. + +/* static */ int32_t Instance::elemDrop(Instance* instance, uint32_t segIndex) { + MOZ_ASSERT(SASigElemDrop.failureMode == FailureMode::FailOnNegI32); + + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveElemSegments_.length(), + "ensured by validation"); + + if (!instance->passiveElemSegments_[segIndex]) { + return 0; + } + + SharedElemSegment& segRefPtr = instance->passiveElemSegments_[segIndex]; + MOZ_RELEASE_ASSERT(!segRefPtr->active()); + + // Drop this instance's reference to the ElemSegment so it can be released. + segRefPtr = nullptr; + return 0; +} + +/* static */ int32_t Instance::dataDrop(Instance* instance, uint32_t segIndex) { + MOZ_ASSERT(SASigDataDrop.failureMode == FailureMode::FailOnNegI32); + + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveDataSegments_.length(), + "ensured by validation"); + + if (!instance->passiveDataSegments_[segIndex]) { + return 0; + } + + SharedDataSegment& segRefPtr = instance->passiveDataSegments_[segIndex]; + MOZ_RELEASE_ASSERT(!segRefPtr->active()); + + // Drop this instance's reference to the DataSegment so it can be released. + segRefPtr = nullptr; + return 0; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Object support. + +/* static */ void Instance::postBarrier(Instance* instance, + gc::Cell** location) { + MOZ_ASSERT(SASigPostBarrier.failureMode == FailureMode::Infallible); + MOZ_ASSERT(location); + instance->storeBuffer_->putCell(reinterpret_cast(location)); +} + +/* static */ void Instance::postBarrierPrecise(Instance* instance, + JSObject** location, + JSObject* prev) { + MOZ_ASSERT(SASigPostBarrierPrecise.failureMode == FailureMode::Infallible); + postBarrierPreciseWithOffset(instance, location, /*offset=*/0, prev); +} + +/* static */ void Instance::postBarrierPreciseWithOffset(Instance* instance, + JSObject** base, + uint32_t offset, + JSObject* prev) { + MOZ_ASSERT(SASigPostBarrierPreciseWithOffset.failureMode == + FailureMode::Infallible); + MOZ_ASSERT(base); + JSObject** location = (JSObject**)(uintptr_t(base) + size_t(offset)); + JSObject* next = *location; + JSObject::postWriteBarrier(location, prev, next); +} + +////////////////////////////////////////////////////////////////////////////// +// +// GC and exception handling support. + +/* static */ void* Instance::structNew(Instance* instance, + TypeDefInstanceData* typeDefData) { + MOZ_ASSERT(SASigStructNew.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + // The new struct will be allocated in an initial heap as determined by + // pretenuring logic as set up in `Instance::init`. + return WasmStructObject::createStruct( + cx, typeDefData, typeDefData->allocSite.initialHeap()); +} + +/* static */ void* Instance::structNewUninit(Instance* instance, + TypeDefInstanceData* typeDefData) { + MOZ_ASSERT(SASigStructNew.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + // The new struct will be allocated in an initial heap as determined by + // pretenuring logic as set up in `Instance::init`. + return WasmStructObject::createStruct( + cx, typeDefData, typeDefData->allocSite.initialHeap()); +} + +/* static */ void* Instance::arrayNew(Instance* instance, uint32_t numElements, + TypeDefInstanceData* typeDefData) { + MOZ_ASSERT(SASigArrayNew.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + // The new array will be allocated in an initial heap as determined by + // pretenuring logic as set up in `Instance::init`. + return WasmArrayObject::createArray( + cx, typeDefData, typeDefData->allocSite.initialHeap(), numElements); +} + +/* static */ void* Instance::arrayNewUninit(Instance* instance, + uint32_t numElements, + TypeDefInstanceData* typeDefData) { + MOZ_ASSERT(SASigArrayNew.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + // The new array will be allocated in an initial heap as determined by + // pretenuring logic as set up in `Instance::init`. + return WasmArrayObject::createArray( + cx, typeDefData, typeDefData->allocSite.initialHeap(), numElements); +} + +// Creates an array (WasmArrayObject) containing `numElements` of type +// described by `typeDef`. Initialises it with data copied from the data +// segment whose index is `segIndex`, starting at byte offset `segByteOffset` +// in the segment. Traps if the segment doesn't hold enough bytes to fill the +// array. +/* static */ void* Instance::arrayNewData(Instance* instance, + uint32_t segByteOffset, + uint32_t numElements, + TypeDefInstanceData* typeDefData, + uint32_t segIndex) { + MOZ_ASSERT(SASigArrayNewData.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + + // Check that the data segment is valid for use. + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveDataSegments_.length(), + "ensured by validation"); + const DataSegment* seg = instance->passiveDataSegments_[segIndex]; + + // `seg` will be nullptr if the segment has already been 'data.drop'ed + // (either implicitly in the case of 'active' segments during instantiation, + // or explicitly by the data.drop instruction.) In that case we can + // continue only if there's no need to copy any data out of it. + if (!seg && (numElements != 0 || segByteOffset != 0)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return nullptr; + } + // At this point, if `seg` is null then `numElements` and `segByteOffset` + // are both zero. + + const TypeDef* typeDef = typeDefData->typeDef; + Rooted arrayObj( + cx, + WasmArrayObject::createArray( + cx, typeDefData, typeDefData->allocSite.initialHeap(), numElements)); + if (!arrayObj) { + // WasmArrayObject::createArray will have reported OOM. + return nullptr; + } + MOZ_RELEASE_ASSERT(arrayObj->is()); + + if (!seg) { + // A zero-length array was requested and has been created, so we're done. + return arrayObj; + } + + // Compute the number of bytes to copy, ensuring it's below 2^32. + CheckedUint32 numBytesToCopy = + CheckedUint32(numElements) * + CheckedUint32(typeDef->arrayType().elementType_.size()); + if (!numBytesToCopy.isValid()) { + // Because the request implies that 2^32 or more bytes are to be copied. + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return nullptr; + } + + // Range-check the copy. The obvious thing to do is to compute the offset + // of the last byte to copy, but that would cause underflow in the + // zero-length-and-zero-offset case. Instead, compute that value plus one; + // in other words the offset of the first byte *not* to copy. + CheckedUint32 lastByteOffsetPlus1 = + CheckedUint32(segByteOffset) + numBytesToCopy; + + CheckedUint32 numBytesAvailable(seg->bytes.length()); + if (!lastByteOffsetPlus1.isValid() || !numBytesAvailable.isValid() || + lastByteOffsetPlus1.value() > numBytesAvailable.value()) { + // Because the last byte to copy doesn't exist inside `seg->bytes`. + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return nullptr; + } + + // Because `numBytesToCopy` is an in-range `CheckedUint32`, the cast to + // `size_t` is safe even on a 32-bit target. + memcpy(arrayObj->data_, &seg->bytes[segByteOffset], + size_t(numBytesToCopy.value())); + + return arrayObj; +} + +// This is almost identical to ::arrayNewData, apart from the final part that +// actually copies the data. It creates an array (WasmArrayObject) +// containing `numElements` of type described by `typeDef`. Initialises it +// with data copied from the element segment whose index is `segIndex`, +// starting at element number `segElemIndex` in the segment. Traps if the +// segment doesn't hold enough elements to fill the array. +/* static */ void* Instance::arrayNewElem(Instance* instance, + uint32_t segElemIndex, + uint32_t numElements, + TypeDefInstanceData* typeDefData, + uint32_t segIndex) { + MOZ_ASSERT(SASigArrayNewElem.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + + // Check that the element segment is valid for use. + MOZ_RELEASE_ASSERT(size_t(segIndex) < instance->passiveElemSegments_.length(), + "ensured by validation"); + const ElemSegment* seg = instance->passiveElemSegments_[segIndex]; + + // As with ::arrayNewData, if `seg` is nullptr then we can only safely copy + // zero elements from it. + if (!seg && (numElements != 0 || segElemIndex != 0)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return nullptr; + } + // At this point, if `seg` is null then `numElements` and `segElemIndex` + // are both zero. + + const TypeDef* typeDef = typeDefData->typeDef; + + // The element segment is an array of uint32_t indicating function indices, + // which we'll have to dereference (to produce real function pointers) + // before parking them in the array. Hence each array element must be a + // machine word. + MOZ_RELEASE_ASSERT(typeDef->arrayType().elementType_.size() == sizeof(void*)); + + Rooted arrayObj( + cx, + WasmArrayObject::createArray( + cx, typeDefData, typeDefData->allocSite.initialHeap(), numElements)); + if (!arrayObj) { + // WasmArrayObject::createArray will have reported OOM. + return nullptr; + } + MOZ_RELEASE_ASSERT(arrayObj->is()); + + if (!seg) { + // A zero-length array was requested and has been created, so we're done. + return arrayObj; + } + + // Range-check the copy. As in ::arrayNewData, compute the index of the + // last element to copy, plus one. + CheckedUint32 lastIndexPlus1 = + CheckedUint32(segElemIndex) + CheckedUint32(numElements); + + CheckedUint32 numElemsAvailable(seg->elemFuncIndices.length()); + if (!lastIndexPlus1.isValid() || !numElemsAvailable.isValid() || + lastIndexPlus1.value() > numElemsAvailable.value()) { + // Because the last element to copy doesn't exist inside + // `seg->elemFuncIndices`. + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return nullptr; + } + + // Do the initialisation, converting function indices into code pointers as + // we go. + void** dst = (void**)arrayObj->data_; + const uint32_t* src = &seg->elemFuncIndices[segElemIndex]; + for (uint32_t i = 0; i < numElements; i++) { + uint32_t funcIndex = src[i]; + FieldType elemType = typeDef->arrayType().elementType_; + MOZ_RELEASE_ASSERT(elemType.isRefType()); + RootedVal value(cx, elemType.refType()); + if (funcIndex == NullFuncIndex) { + // value remains null + } else { + void* funcRef = Instance::refFunc(instance, funcIndex); + if (funcRef == AnyRef::invalid().forCompiledCode()) { + return nullptr; // OOM, which has already been reported. + } + value = Val(elemType.refType(), FuncRef::fromCompiledCode(funcRef)); + } + value.get().writeToHeapLocation(&dst[i]); + } + + return arrayObj; +} + +/* static */ int32_t Instance::arrayCopy(Instance* instance, void* dstArray, + uint32_t dstIndex, void* srcArray, + uint32_t srcIndex, + uint32_t numElements, + uint32_t elementSize) { + MOZ_ASSERT(SASigArrayCopy.failureMode == FailureMode::FailOnNegI32); + JSContext* cx = instance->cx(); + + // At the entry point, `elementSize` may be negative to indicate + // reftyped-ness of array elements. That is done in order to avoid having + // to pass yet another (boolean) parameter here. + + // "traps if either array is null" + if (!srcArray || !dstArray) { + ReportTrapError(cx, JSMSG_WASM_DEREF_NULL); + return -1; + } + + bool elemsAreRefTyped = false; + if (int32_t(elementSize) < 0) { + elemsAreRefTyped = true; + elementSize = uint32_t(-int32_t(elementSize)); + } + MOZ_ASSERT(elementSize >= 1 && elementSize <= 16); + + // Get hold of the two arrays. + Rooted dstArrayObj(cx, + static_cast(dstArray)); + MOZ_RELEASE_ASSERT(dstArrayObj->is()); + + Rooted srcArrayObj(cx, + static_cast(srcArray)); + MOZ_RELEASE_ASSERT(srcArrayObj->is()); + + // If WasmArrayObject::numElements() is changed to return 64 bits, the + // following checking logic will be incorrect. + STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32; + + // "traps if destination + length > len(array1)" + uint64_t dstNumElements = uint64_t(dstArrayObj->numElements_); + if (uint64_t(dstIndex) + uint64_t(numElements) > dstNumElements) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + // "traps if source + length > len(array2)" + uint64_t srcNumElements = uint64_t(srcArrayObj->numElements_); + if (uint64_t(srcIndex) + uint64_t(numElements) > srcNumElements) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + // trap if we're asked to copy 2^32 or more bytes on a 32-bit target. + uint64_t numBytesToCopy = uint64_t(numElements) * uint64_t(elementSize); +#ifndef JS_64BIT + if (numBytesToCopy > uint64_t(UINT32_MAX)) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } +#endif + // We're now assured that `numBytesToCopy` can be cast to `size_t` without + // overflow. + + // Actually do the copy, taking care to handle cases where the src and dst + // areas overlap. + uint8_t* srcBase = srcArrayObj->data_; + uint8_t* dstBase = dstArrayObj->data_; + srcBase += size_t(srcIndex) * size_t(elementSize); + dstBase += size_t(dstIndex) * size_t(elementSize); + + if (numBytesToCopy == 0 || srcBase == dstBase) { + // Early exit if there's no work to do. + return 0; + } + + if (!elemsAreRefTyped) { + // Hand off to memmove, which is presumably highly optimized. + memmove(dstBase, srcBase, size_t(numBytesToCopy)); + return 0; + } + + // We're copying refs; doing that needs suitable GC barrier-ing. + uint8_t* nextSrc; + uint8_t* nextDst; + intptr_t step; + if (dstBase < srcBase) { + // Moving data backwards in the address space; so iterate forwards through + // the array. + step = intptr_t(elementSize); + nextSrc = srcBase; + nextDst = dstBase; + } else { + // Moving data forwards; so iterate backwards. + step = -intptr_t(elementSize); + nextSrc = srcBase + size_t(numBytesToCopy) - size_t(elementSize); + nextDst = dstBase + size_t(numBytesToCopy) - size_t(elementSize); + } + // We don't know the type of the elems, only that they are refs. No matter, + // we can simply make up a type. + RefType aRefType = RefType::eq(); + // Do the iteration + for (size_t i = 0; i < size_t(numElements); i++) { + // Copy `elementSize` bytes from `nextSrc` to `nextDst`. + RootedVal value(cx, aRefType); + value.get().readFromHeapLocation(nextSrc); + value.get().writeToHeapLocation(nextDst); + nextSrc += step; + nextDst += step; + } + + return 0; +} + +/* static */ void* Instance::exceptionNew(Instance* instance, JSObject* tag) { + MOZ_ASSERT(SASigExceptionNew.failureMode == FailureMode::FailOnNullPtr); + JSContext* cx = instance->cx(); + Rooted tagObj(cx, &tag->as()); + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmException)); + RootedObject stack(cx, nullptr); + return AnyRef::fromJSObject( + WasmExceptionObject::create(cx, tagObj, stack, proto)) + .forCompiledCode(); +} + +/* static */ int32_t Instance::throwException(Instance* instance, + JSObject* exn) { + MOZ_ASSERT(SASigThrowException.failureMode == FailureMode::FailOnNegI32); + + JSContext* cx = instance->cx(); + RootedValue exnVal(cx, UnboxAnyRef(AnyRef::fromJSObject(exn))); + cx->setPendingException(exnVal, nullptr); + + // By always returning -1, we trigger a wasmTrap(Trap::ThrowReported), + // and use that to trigger the stack walking for this exception. + return -1; +} + +/* static */ int32_t Instance::intrI8VecMul(Instance* instance, uint32_t dest, + uint32_t src1, uint32_t src2, + uint32_t len, uint8_t* memBase) { + MOZ_ASSERT(SASigIntrI8VecMul.failureMode == FailureMode::FailOnNegI32); + + JSContext* cx = instance->cx(); + const WasmArrayRawBuffer* rawBuf = WasmArrayRawBuffer::fromDataPtr(memBase); + size_t memLen = rawBuf->byteLength(); + + // Bounds check and deal with arithmetic overflow. + uint64_t destLimit = uint64_t(dest) + uint64_t(len); + uint64_t src1Limit = uint64_t(src1) + uint64_t(len); + uint64_t src2Limit = uint64_t(src2) + uint64_t(len); + if (destLimit > memLen || src1Limit > memLen || src2Limit > memLen) { + ReportTrapError(cx, JSMSG_WASM_OUT_OF_BOUNDS); + return -1; + } + + // Basic dot product + uint8_t* destPtr = &memBase[dest]; + uint8_t* src1Ptr = &memBase[src1]; + uint8_t* src2Ptr = &memBase[src2]; + while (len > 0) { + *destPtr = (*src1Ptr) * (*src2Ptr); + + destPtr++; + src1Ptr++; + src2Ptr++; + len--; + } + + return 0; +} + +////////////////////////////////////////////////////////////////////////////// +// +// Instance creation and related. + +Instance::Instance(JSContext* cx, Handle object, + const SharedCode& code, Handle memory, + SharedTableVector&& tables, UniqueDebugState maybeDebug) + : realm_(cx->realm()), + jsJitArgsRectifier_( + cx->runtime()->jitRuntime()->getArgumentsRectifier().value), + jsJitExceptionHandler_( + cx->runtime()->jitRuntime()->getExceptionTail().value), + preBarrierCode_( + cx->runtime()->jitRuntime()->preBarrier(MIRType::Object).value), + storeBuffer_(&cx->runtime()->gc.storeBuffer()), + object_(object), + code_(std::move(code)), + memory_(memory), + tables_(std::move(tables)), + maybeDebug_(std::move(maybeDebug)), + debugFilter_(nullptr), + maxInitializedGlobalsIndexPlus1_(0) {} + +Instance* Instance::create(JSContext* cx, Handle object, + const SharedCode& code, uint32_t instanceDataLength, + Handle memory, + SharedTableVector&& tables, + UniqueDebugState maybeDebug) { + void* base = js_calloc(alignof(Instance) + offsetof(Instance, data_) + + instanceDataLength); + if (!base) { + ReportOutOfMemory(cx); + return nullptr; + } + void* aligned = (void*)AlignBytes(uintptr_t(base), alignof(Instance)); + + auto* instance = new (aligned) Instance( + cx, object, code, memory, std::move(tables), std::move(maybeDebug)); + instance->allocatedBase_ = base; + return instance; +} + +void Instance::destroy(Instance* instance) { + instance->~Instance(); + js_free(instance->allocatedBase_); +} + +bool Instance::init(JSContext* cx, const JSObjectVector& funcImports, + const ValVector& globalImportValues, + const WasmGlobalObjectVector& globalObjs, + const WasmTagObjectVector& tagObjs, + const DataSegmentVector& dataSegments, + const ElemSegmentVector& elemSegments) { + MOZ_ASSERT(!!maybeDebug_ == metadata().debugEnabled); + +#ifdef DEBUG + for (auto t : code_->tiers()) { + MOZ_ASSERT(funcImports.length() == metadata(t).funcImports.length()); + } +#endif + MOZ_ASSERT(tables_.length() == metadata().tables.length()); + + memoryBase_ = + memory_ ? memory_->buffer().dataPointerEither().unwrap() : nullptr; + size_t limit = memory_ ? memory_->boundsCheckLimit() : 0; +#if !defined(JS_64BIT) + // We assume that the limit is a 32-bit quantity + MOZ_ASSERT(limit <= UINT32_MAX); +#endif + boundsCheckLimit_ = limit; + cx_ = cx; + valueBoxClass_ = &WasmValueBox::class_; + resetInterrupt(cx); + jumpTable_ = code_->tieringJumpTable(); + debugFilter_ = nullptr; + addressOfNeedsIncrementalBarrier_ = + cx->compartment()->zone()->addressOfNeedsIncrementalBarrier(); + + // Initialize function imports in the instance data + Tier callerTier = code_->bestTier(); + for (size_t i = 0; i < metadata(callerTier).funcImports.length(); i++) { + JSObject* f = funcImports[i]; + MOZ_ASSERT(f->isCallable()); + const FuncImport& fi = metadata(callerTier).funcImports[i]; + const FuncType& funcType = metadata().getFuncImportType(fi); + FuncImportInstanceData& import = funcImportInstanceData(fi); + import.callable = f; + if (f->is()) { + JSFunction* fun = &f->as(); + if (!isAsmJS() && IsWasmExportedFunction(fun)) { + WasmInstanceObject* calleeInstanceObj = + ExportedFunctionToInstanceObject(fun); + Instance& calleeInstance = calleeInstanceObj->instance(); + Tier calleeTier = calleeInstance.code().bestTier(); + const CodeRange& codeRange = + calleeInstanceObj->getExportedFunctionCodeRange( + &f->as(), calleeTier); + import.instance = &calleeInstance; + import.realm = fun->realm(); + import.code = calleeInstance.codeBase(calleeTier) + + codeRange.funcUncheckedCallEntry(); + } else if (void* thunk = MaybeGetBuiltinThunk(fun, funcType)) { + import.instance = this; + import.realm = fun->realm(); + import.code = thunk; + } else { + import.instance = this; + import.realm = fun->realm(); + import.code = codeBase(callerTier) + fi.interpExitCodeOffset(); + } + } else { + import.instance = this; + import.realm = f->nonCCWRealm(); + import.code = codeBase(callerTier) + fi.interpExitCodeOffset(); + } + } + + // Initialize tables in the instance data + for (size_t i = 0; i < tables_.length(); i++) { + const TableDesc& td = metadata().tables[i]; + TableInstanceData& table = tableInstanceData(i); + table.length = tables_[i]->length(); + table.elements = tables_[i]->instanceElements(); + // Non-imported tables, with init_expr, has to be initialized with + // the evaluated value. + if (!td.isImported && td.initExpr) { + Rooted instanceObj(cx, object()); + RootedVal val(cx); + if (!td.initExpr->evaluate(cx, instanceObj, &val)) { + return false; + } + RootedAnyRef ref(cx, val.get().ref()); + tables_[i]->fillUninitialized(0, tables_[i]->length(), ref, cx); + } + } + +#ifdef DEBUG + // All (linked) tables with non-nullable types must be initialized. + for (size_t i = 0; i < tables_.length(); i++) { + const TableDesc& td = metadata().tables[i]; + if (!td.elemType.isNullable()) { + tables_[i]->assertRangeNotNull(0, tables_[i]->length()); + } + } +#endif // DEBUG + + // Initialize tags in the instance data + for (size_t i = 0; i < metadata().tags.length(); i++) { + MOZ_ASSERT(tagObjs[i] != nullptr); + tagInstanceData(i).object = tagObjs[i]; + } + pendingException_ = nullptr; + pendingExceptionTag_ = nullptr; + + // Add debug filtering table. + if (metadata().debugEnabled) { + size_t numFuncs = metadata().debugNumFuncs(); + size_t numWords = std::max((numFuncs + 31) / 32, 1); + debugFilter_ = (uint32_t*)js_calloc(numWords, sizeof(uint32_t)); + if (!debugFilter_) { + return false; + } + } + + // Add observer if our memory base may grow + if (memory_ && memory_->movingGrowable() && + !memory_->addMovingGrowObserver(cx, object_)) { + return false; + } + + // Add observers if our tables may grow + for (const SharedTable& table : tables_) { + if (table->movingGrowable() && !table->addMovingGrowObserver(cx, object_)) { + return false; + } + } + + // Initialize type definitions in the instance data. + const SharedTypeContext& types = metadata().types; + Zone* zone = realm()->zone(); + for (uint32_t typeIndex = 0; typeIndex < types->length(); typeIndex++) { + const TypeDef& typeDef = types->type(typeIndex); + TypeDefInstanceData* typeDefData = typeDefInstanceData(typeIndex); + + // Set default field values. + new (typeDefData) TypeDefInstanceData(); + + // Store the runtime type for this type index + typeDefData->typeDef = &typeDef; + typeDefData->superTypeVector = typeDef.superTypeVector(); + + if (typeDef.kind() == TypeDefKind::Struct || + typeDef.kind() == TypeDefKind::Array) { + // Compute the parameters that allocation will use. First, the class + // and alloc kind for the type definition. + const JSClass* clasp; + gc::AllocKind allocKind; + + if (typeDef.kind() == TypeDefKind::Struct) { + clasp = WasmStructObject::classForTypeDef(&typeDef); + allocKind = WasmStructObject::allocKindForTypeDef(&typeDef); + } else { + clasp = &WasmArrayObject::class_; + allocKind = WasmArrayObject::allocKind(); + } + + // Move the alloc kind to background if possible + if (CanChangeToBackgroundAllocKind(allocKind, clasp)) { + allocKind = ForegroundToBackgroundAllocKind(allocKind); + } + + // Find the shape using the class and recursion group + typeDefData->shape = + WasmGCShape::getShape(cx, clasp, cx->realm(), TaggedProto(), + &typeDef.recGroup(), ObjectFlags()); + if (!typeDefData->shape) { + return false; + } + + typeDefData->clasp = clasp; + typeDefData->allocKind = allocKind; + + // Initialize the allocation site for pre-tenuring. + typeDefData->allocSite.initWasm(zone); + } else if (typeDef.kind() == TypeDefKind::Func) { + // Nothing to do; the default values are OK. + } else { + MOZ_ASSERT(typeDef.kind() == TypeDefKind::None); + MOZ_CRASH(); + } + } + + // Initialize globals in the instance data. + // + // This must be performed after we have initialized runtime types as a global + // initializer may reference them. + // + // We increment `maxInitializedGlobalsIndexPlus1_` every iteration of the + // loop, as we call out to `InitExpr::evaluate` which may call + // `constantGlobalGet` which uses this value to assert we're never accessing + // uninitialized globals. + maxInitializedGlobalsIndexPlus1_ = 0; + for (size_t i = 0; i < metadata().globals.length(); + i++, maxInitializedGlobalsIndexPlus1_ = i) { + const GlobalDesc& global = metadata().globals[i]; + + // Constants are baked into the code, never stored in the global area. + if (global.isConstant()) { + continue; + } + + uint8_t* globalAddr = data() + global.offset(); + switch (global.kind()) { + case GlobalKind::Import: { + size_t imported = global.importIndex(); + if (global.isIndirect()) { + *(void**)globalAddr = + (void*)&globalObjs[imported]->val().get().cell(); + } else { + globalImportValues[imported].writeToHeapLocation(globalAddr); + } + break; + } + case GlobalKind::Variable: { + RootedVal val(cx); + const InitExpr& init = global.initExpr(); + Rooted instanceObj(cx, object()); + if (!init.evaluate(cx, instanceObj, &val)) { + return false; + } + + if (global.isIndirect()) { + // Initialize the cell + wasm::GCPtrVal& cell = globalObjs[i]->val(); + cell = val.get(); + // Link to the cell + void* address = (void*)&cell.get().cell(); + *(void**)globalAddr = address; + } else { + val.get().writeToHeapLocation(globalAddr); + } + break; + } + case GlobalKind::Constant: { + MOZ_CRASH("skipped at the top"); + } + } + } + + // All globals were initialized + MOZ_ASSERT(maxInitializedGlobalsIndexPlus1_ == metadata().globals.length()); + + // Take references to the passive data segments + if (!passiveDataSegments_.resize(dataSegments.length())) { + return false; + } + for (size_t i = 0; i < dataSegments.length(); i++) { + if (!dataSegments[i]->active()) { + passiveDataSegments_[i] = dataSegments[i]; + } + } + + // Take references to the passive element segments + if (!passiveElemSegments_.resize(elemSegments.length())) { + return false; + } + for (size_t i = 0; i < elemSegments.length(); i++) { + if (elemSegments[i]->kind == ElemSegment::Kind::Passive) { + passiveElemSegments_[i] = elemSegments[i]; + } + } + + return true; +} + +Instance::~Instance() { + realm_->wasm.unregisterInstance(*this); + + if (debugFilter_) { + js_free(debugFilter_); + } + + // Any pending exceptions should have been consumed. + MOZ_ASSERT(!pendingException_); +} + +void Instance::setInterrupt() { + interrupt_ = true; + stackLimit_ = JS::NativeStackLimitMin; +} + +bool Instance::isInterrupted() const { + return interrupt_ || stackLimit_ == JS::NativeStackLimitMin; +} + +void Instance::resetInterrupt(JSContext* cx) { + interrupt_ = false; + stackLimit_ = cx->stackLimitForJitCode(JS::StackForUntrustedScript); +} + +bool Instance::debugFilter(uint32_t funcIndex) const { + return (debugFilter_[funcIndex / 32] >> funcIndex % 32) & 1; +} + +void Instance::setDebugFilter(uint32_t funcIndex, bool value) { + if (value) { + debugFilter_[funcIndex / 32] |= (1 << funcIndex % 32); + } else { + debugFilter_[funcIndex / 32] &= ~(1 << funcIndex % 32); + } +} + +size_t Instance::memoryMappedSize() const { + return memory_->buffer().wasmMappedSize(); +} + +bool Instance::memoryAccessInGuardRegion(const uint8_t* addr, + unsigned numBytes) const { + MOZ_ASSERT(numBytes > 0); + + if (!metadata().usesMemory()) { + return false; + } + + uint8_t* base = memoryBase().unwrap(/* comparison */); + if (addr < base) { + return false; + } + + size_t lastByteOffset = addr - base + (numBytes - 1); + return lastByteOffset >= memory()->volatileMemoryLength() && + lastByteOffset < memoryMappedSize(); +} + +void Instance::tracePrivate(JSTracer* trc) { + // This method is only called from WasmInstanceObject so the only reason why + // TraceEdge is called is so that the pointer can be updated during a moving + // GC. + MOZ_ASSERT_IF(trc->isMarkingTracer(), gc::IsMarked(trc->runtime(), object_)); + TraceEdge(trc, &object_, "wasm instance object"); + + // OK to just do one tier here; though the tiers have different funcImports + // tables, they share the instance object. + for (const FuncImport& fi : metadata(code().stableTier()).funcImports) { + TraceNullableEdge(trc, &funcImportInstanceData(fi).callable, "wasm import"); + } + + for (const SharedTable& table : tables_) { + table->trace(trc); + } + + for (const GlobalDesc& global : code().metadata().globals) { + // Indirect reference globals get traced by the owning WebAssembly.Global. + if (!global.type().isRefRepr() || global.isConstant() || + global.isIndirect()) { + continue; + } + GCPtr* obj = (GCPtr*)(data() + global.offset()); + TraceNullableEdge(trc, obj, "wasm reference-typed global"); + } + + for (uint32_t tagIndex = 0; tagIndex < code().metadata().tags.length(); + tagIndex++) { + TraceNullableEdge(trc, &tagInstanceData(tagIndex).object, "wasm tag"); + } + + const SharedTypeContext& types = metadata().types; + for (uint32_t typeIndex = 0; typeIndex < types->length(); typeIndex++) { + TypeDefInstanceData* typeDefData = typeDefInstanceData(typeIndex); + TraceNullableEdge(trc, &typeDefData->shape, "wasm shape"); + } + + TraceNullableEdge(trc, &memory_, "wasm buffer"); + TraceNullableEdge(trc, &pendingException_, "wasm pending exception value"); + TraceNullableEdge(trc, &pendingExceptionTag_, "wasm pending exception tag"); + + if (maybeDebug_) { + maybeDebug_->trace(trc); + } +} + +void js::wasm::TraceInstanceEdge(JSTracer* trc, Instance* instance, + const char* name) { + if (IsTracerKind(trc, JS::TracerKind::Moving)) { + // Compacting GC: The Instance does not move so there is nothing to do here. + // Reading the object from the instance below would be a data race during + // multi-threaded updates. Compacting GC does not rely on graph traversal + // to find all edges that need to be updated. + return; + } + + // Instance fields are traced by the owning WasmInstanceObject's trace + // hook. Tracing this ensures they are traced once. + JSObject* object = instance->objectUnbarriered(); + TraceManuallyBarrieredEdge(trc, &object, name); +} + +uintptr_t Instance::traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi, + uint8_t* nextPC, + uintptr_t highestByteVisitedInPrevFrame) { + const StackMap* map = code().lookupStackMap(nextPC); + if (!map) { + return 0; + } + + Frame* frame = wfi.frame(); + + // |frame| points somewhere in the middle of the area described by |map|. + // We have to calculate |scanStart|, the lowest address that is described by + // |map|, by consulting |map->frameOffsetFromTop|. + + const size_t numMappedBytes = map->header.numMappedWords * sizeof(void*); + const uintptr_t scanStart = uintptr_t(frame) + + (map->header.frameOffsetFromTop * sizeof(void*)) - + numMappedBytes; + MOZ_ASSERT(0 == scanStart % sizeof(void*)); + + // Do what we can to assert that, for consecutive wasm frames, their stack + // maps also abut exactly. This is a useful sanity check on the sizing of + // stackmaps. + // + // In debug builds, the stackmap construction machinery goes to considerable + // efforts to ensure that the stackmaps for consecutive frames abut exactly. + // This is so as to ensure there are no areas of stack inadvertently ignored + // by a stackmap, nor covered by two stackmaps. Hence any failure of this + // assertion is serious and should be investigated. +#ifndef JS_CODEGEN_ARM64 + MOZ_ASSERT_IF(highestByteVisitedInPrevFrame != 0, + highestByteVisitedInPrevFrame + 1 == scanStart); +#endif + + uintptr_t* stackWords = (uintptr_t*)scanStart; + + // If we have some exit stub words, this means the map also covers an area + // created by a exit stub, and so the highest word of that should be a + // constant created by (code created by) GenerateTrapExit. + MOZ_ASSERT_IF( + map->header.numExitStubWords > 0, + stackWords[map->header.numExitStubWords - 1 - + TrapExitDummyValueOffsetFromTop] == TrapExitDummyValue); + + // And actually hand them off to the GC. + for (uint32_t i = 0; i < map->header.numMappedWords; i++) { + if (map->getBit(i) == 0) { + continue; + } + + // TODO/AnyRef-boxing: With boxed immediates and strings, the value may + // not be a traceable JSObject*. + ASSERT_ANYREF_IS_JSOBJECT; + + // This assertion seems at least moderately effective in detecting + // discrepancies or misalignments between the map and reality. + MOZ_ASSERT(js::gc::IsCellPointerValidOrNull((const void*)stackWords[i])); + + if (stackWords[i]) { + TraceRoot(trc, (JSObject**)&stackWords[i], + "Instance::traceWasmFrame: normal word"); + } + } + + // Finally, deal with any GC-managed fields in the DebugFrame, if it is + // present and those fields may be live. + if (map->header.hasDebugFrameWithLiveRefs) { + DebugFrame* debugFrame = DebugFrame::from(frame); + char* debugFrameP = (char*)debugFrame; + + // TODO/AnyRef-boxing: With boxed immediates and strings, the value may + // not be a traceable JSObject*. + ASSERT_ANYREF_IS_JSOBJECT; + + for (size_t i = 0; i < MaxRegisterResults; i++) { + if (debugFrame->hasSpilledRegisterRefResult(i)) { + char* resultRefP = debugFrameP + DebugFrame::offsetOfRegisterResult(i); + TraceNullableRoot( + trc, (JSObject**)resultRefP, + "Instance::traceWasmFrame: DebugFrame::resultResults_"); + } + } + + if (debugFrame->hasCachedReturnJSValue()) { + char* cachedReturnJSValueP = + debugFrameP + DebugFrame::offsetOfCachedReturnJSValue(); + TraceRoot(trc, (js::Value*)cachedReturnJSValueP, + "Instance::traceWasmFrame: DebugFrame::cachedReturnJSValue_"); + } + } + + return scanStart + numMappedBytes - 1; +} + +WasmMemoryObject* Instance::memory() const { return memory_; } + +SharedMem Instance::memoryBase() const { + MOZ_ASSERT(metadata().usesMemory()); + MOZ_ASSERT(memoryBase_ == memory_->buffer().dataPointerEither()); + return memory_->buffer().dataPointerEither(); +} + +SharedArrayRawBuffer* Instance::sharedMemoryBuffer() const { + MOZ_ASSERT(memory_->isShared()); + return memory_->sharedArrayRawBuffer(); +} + +WasmInstanceObject* Instance::objectUnbarriered() const { + return object_.unbarrieredGet(); +} + +WasmInstanceObject* Instance::object() const { return object_; } + +static bool EnsureEntryStubs(const Instance& instance, uint32_t funcIndex, + const FuncExport** funcExport, + void** interpEntry) { + Tier tier = instance.code().bestTier(); + + size_t funcExportIndex; + *funcExport = + &instance.metadata(tier).lookupFuncExport(funcIndex, &funcExportIndex); + + const FuncExport& fe = **funcExport; + if (fe.hasEagerStubs()) { + *interpEntry = instance.codeBase(tier) + fe.eagerInterpEntryOffset(); + return true; + } + + MOZ_ASSERT(!instance.isAsmJS(), "only wasm can lazily export functions"); + + // If the best tier is Ion, life is simple: background compilation has + // already completed and has been committed, so there's no risk of race + // conditions here. + // + // If the best tier is Baseline, there could be a background compilation + // happening at the same time. The background compilation will lock the + // first tier lazy stubs first to stop new baseline stubs from being + // generated, then the second tier stubs to generate them. + // + // - either we take the tier1 lazy stub lock before the background + // compilation gets it, then we generate the lazy stub for tier1. When the + // background thread gets the tier1 lazy stub lock, it will see it has a + // lazy stub and will recompile it for tier2. + // - or we don't take the lock here first. Background compilation won't + // find a lazy stub for this function, thus won't generate it. So we'll do + // it ourselves after taking the tier2 lock. + // + // Also see doc block for stubs in WasmJS.cpp. + + auto stubs = instance.code(tier).lazyStubs().writeLock(); + *interpEntry = stubs->lookupInterpEntry(fe.funcIndex()); + if (*interpEntry) { + return true; + } + + // The best tier might have changed after we've taken the lock. + Tier prevTier = tier; + tier = instance.code().bestTier(); + const Metadata& metadata = instance.metadata(); + const CodeTier& codeTier = instance.code(tier); + if (tier == prevTier) { + if (!stubs->createOneEntryStub(funcExportIndex, metadata, codeTier)) { + return false; + } + + *interpEntry = stubs->lookupInterpEntry(fe.funcIndex()); + MOZ_ASSERT(*interpEntry); + return true; + } + + MOZ_RELEASE_ASSERT(prevTier == Tier::Baseline && tier == Tier::Optimized); + auto stubs2 = instance.code(tier).lazyStubs().writeLock(); + + // If it didn't have a stub in the first tier, background compilation + // shouldn't have made one in the second tier. + MOZ_ASSERT(!stubs2->hasEntryStub(fe.funcIndex())); + + if (!stubs2->createOneEntryStub(funcExportIndex, metadata, codeTier)) { + return false; + } + + *interpEntry = stubs2->lookupInterpEntry(fe.funcIndex()); + MOZ_ASSERT(*interpEntry); + return true; +} + +static bool GetInterpEntryAndEnsureStubs(JSContext* cx, Instance& instance, + uint32_t funcIndex, CallArgs args, + void** interpEntry, + const FuncType** funcType) { + const FuncExport* funcExport; + if (!EnsureEntryStubs(instance, funcIndex, &funcExport, interpEntry)) { + return false; + } + + *funcType = &instance.metadata().getFuncExportType(*funcExport); + +#ifdef DEBUG + // EnsureEntryStubs() has ensured proper jit-entry stubs have been created and + // installed in funcIndex's JumpTable entry, so check against the presence of + // the provisional lazy stub. See also + // WasmInstanceObject::getExportedFunction(). + if (!funcExport->hasEagerStubs() && (*funcType)->canHaveJitEntry()) { + if (!EnsureBuiltinThunksInitialized()) { + return false; + } + JSFunction& callee = args.callee().as(); + void* provisionalLazyJitEntryStub = ProvisionalLazyJitEntryStub(); + MOZ_ASSERT(provisionalLazyJitEntryStub); + MOZ_ASSERT(callee.isWasmWithJitEntry()); + MOZ_ASSERT(*callee.wasmJitEntry() != provisionalLazyJitEntryStub); + } +#endif + return true; +} + +bool wasm::ResultsToJSValue(JSContext* cx, ResultType type, + void* registerResultLoc, + Maybe stackResultsLoc, + MutableHandleValue rval, CoercionLevel level) { + if (type.empty()) { + // No results: set to undefined, and we're done. + rval.setUndefined(); + return true; + } + + // If we added support for multiple register results, we'd need to establish a + // convention for how to store them to memory in registerResultLoc. For now + // we can punt. + static_assert(MaxRegisterResults == 1); + + // Stack results written to stackResultsLoc; register result written + // to registerResultLoc. + + // First, convert the register return value, and prepare to iterate in + // push order. Note that if the register result is a reference type, + // it may be unrooted, so ToJSValue_anyref must not GC in that case. + ABIResultIter iter(type); + DebugOnly usedRegisterResult = false; + for (; !iter.done(); iter.next()) { + if (iter.cur().inRegister()) { + MOZ_ASSERT(!usedRegisterResult); + if (!ToJSValue(cx, registerResultLoc, iter.cur().type(), + rval, level)) { + return false; + } + usedRegisterResult = true; + } + } + MOZ_ASSERT(usedRegisterResult); + + MOZ_ASSERT((stackResultsLoc.isSome()) == (iter.count() > 1)); + if (!stackResultsLoc) { + // A single result: we're done. + return true; + } + + // Otherwise, collect results in an array, in push order. + Rooted array(cx, NewDenseEmptyArray(cx)); + if (!array) { + return false; + } + RootedValue tmp(cx); + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + const ABIResult& result = iter.cur(); + if (result.onStack()) { + char* loc = stackResultsLoc.value() + result.stackOffset(); + if (!ToJSValue(cx, loc, result.type(), &tmp, level)) { + return false; + } + if (!NewbornArrayPush(cx, array, tmp)) { + return false; + } + } else { + if (!NewbornArrayPush(cx, array, rval)) { + return false; + } + } + } + rval.set(ObjectValue(*array)); + return true; +} + +class MOZ_RAII ReturnToJSResultCollector { + class MOZ_RAII StackResultsRooter : public JS::CustomAutoRooter { + ReturnToJSResultCollector& collector_; + + public: + StackResultsRooter(JSContext* cx, ReturnToJSResultCollector& collector) + : JS::CustomAutoRooter(cx), collector_(collector) {} + + void trace(JSTracer* trc) final { + for (ABIResultIter iter(collector_.type_); !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + if (result.onStack() && result.type().isRefRepr()) { + char* loc = collector_.stackResultsArea_.get() + result.stackOffset(); + JSObject** refLoc = reinterpret_cast(loc); + TraceNullableRoot(trc, refLoc, "StackResultsRooter::trace"); + } + } + } + }; + friend class StackResultsRooter; + + ResultType type_; + UniquePtr stackResultsArea_; + Maybe rooter_; + + public: + explicit ReturnToJSResultCollector(const ResultType& type) : type_(type){}; + bool init(JSContext* cx) { + bool needRooter = false; + ABIResultIter iter(type_); + for (; !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + if (result.onStack() && result.type().isRefRepr()) { + needRooter = true; + } + } + uint32_t areaBytes = iter.stackBytesConsumedSoFar(); + MOZ_ASSERT_IF(needRooter, areaBytes > 0); + if (areaBytes > 0) { + // It is necessary to zero storage for ref results, and it doesn't + // hurt to do so for other POD results. + stackResultsArea_ = cx->make_zeroed_pod_array(areaBytes); + if (!stackResultsArea_) { + return false; + } + if (needRooter) { + rooter_.emplace(cx, *this); + } + } + return true; + } + + void* stackResultsArea() { + MOZ_ASSERT(stackResultsArea_); + return stackResultsArea_.get(); + } + + bool collect(JSContext* cx, void* registerResultLoc, MutableHandleValue rval, + CoercionLevel level) { + Maybe stackResultsLoc = + stackResultsArea_ ? Some(stackResultsArea_.get()) : Nothing(); + return ResultsToJSValue(cx, type_, registerResultLoc, stackResultsLoc, rval, + level); + } +}; + +bool Instance::callExport(JSContext* cx, uint32_t funcIndex, CallArgs args, + CoercionLevel level) { + if (memory_) { + // If there has been a moving grow, this Instance should have been notified. + MOZ_RELEASE_ASSERT(memory_->buffer().dataPointerEither() == memoryBase()); + } + + void* interpEntry; + const FuncType* funcType; + if (!GetInterpEntryAndEnsureStubs(cx, *this, funcIndex, args, &interpEntry, + &funcType)) { + return false; + } + + // Lossless coercions can handle unexposable arguments or returns. This is + // only available in testing code. + if (level != CoercionLevel::Lossless && funcType->hasUnexposableArgOrRet()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + + ArgTypeVector argTypes(*funcType); + ResultType resultType(ResultType::Vector(funcType->results())); + ReturnToJSResultCollector results(resultType); + if (!results.init(cx)) { + return false; + } + + // The calling convention for an external call into wasm is to pass an + // array of 16-byte values where each value contains either a coerced int32 + // (in the low word), or a double value (in the low dword) value, with the + // coercions specified by the wasm signature. The external entry point + // unpacks this array into the system-ABI-specified registers and stack + // memory and then calls into the internal entry point. The return value is + // stored in the first element of the array (which, therefore, must have + // length >= 1). + Vector exportArgs(cx); + if (!exportArgs.resize( + std::max(1, argTypes.lengthWithStackResults()))) { + return false; + } + + ASSERT_ANYREF_IS_JSOBJECT; + Rooted> refs(cx); + + DebugCodegen(DebugChannel::Function, "wasm-function[%d] arguments [", + funcIndex); + RootedValue v(cx); + for (size_t i = 0; i < argTypes.lengthWithStackResults(); ++i) { + void* rawArgLoc = &exportArgs[i]; + if (argTypes.isSyntheticStackResultPointerArg(i)) { + *reinterpret_cast(rawArgLoc) = results.stackResultsArea(); + continue; + } + size_t naturalIdx = argTypes.naturalIndex(i); + v = naturalIdx < args.length() ? args[naturalIdx] : UndefinedValue(); + ValType type = funcType->arg(naturalIdx); + if (!ToWebAssemblyValue(cx, v, type, rawArgLoc, true, + level)) { + return false; + } + if (type.isRefRepr()) { + void* ptr = *reinterpret_cast(rawArgLoc); + // Store in rooted array until no more GC is possible. + RootedAnyRef ref(cx, AnyRef::fromCompiledCode(ptr)); + ASSERT_ANYREF_IS_JSOBJECT; + if (!refs.emplaceBack(ref.get().asJSObject())) { + return false; + } + DebugCodegen(DebugChannel::Function, "/(#%d)", int(refs.length() - 1)); + } + } + + // Copy over reference values from the rooted array, if any. + if (refs.length() > 0) { + DebugCodegen(DebugChannel::Function, "; "); + size_t nextRef = 0; + for (size_t i = 0; i < argTypes.lengthWithStackResults(); ++i) { + if (argTypes.isSyntheticStackResultPointerArg(i)) { + continue; + } + size_t naturalIdx = argTypes.naturalIndex(i); + ValType type = funcType->arg(naturalIdx); + if (type.isRefRepr()) { + void** rawArgLoc = (void**)&exportArgs[i]; + *rawArgLoc = refs[nextRef++]; + DebugCodegen(DebugChannel::Function, " ref(#%d) := %p ", + int(nextRef - 1), *rawArgLoc); + } + } + refs.clear(); + } + + DebugCodegen(DebugChannel::Function, "]\n"); + + // Ensure pending exception is cleared before and after (below) call. + MOZ_ASSERT(!pendingException_); + + { + JitActivation activation(cx); + + // Call the per-exported-function trampoline created by GenerateEntry. + auto funcPtr = JS_DATA_TO_FUNC_PTR(ExportFuncPtr, interpEntry); + if (!CALL_GENERATED_2(funcPtr, exportArgs.begin(), this)) { + return false; + } + } + + MOZ_ASSERT(!pendingException_); + + if (isAsmJS() && args.isConstructing()) { + // By spec, when a JS function is called as a constructor and this + // function returns a primary type, which is the case for all asm.js + // exported functions, the returned value is discarded and an empty + // object is returned instead. + PlainObject* obj = NewPlainObject(cx); + if (!obj) { + return false; + } + args.rval().set(ObjectValue(*obj)); + return true; + } + + // Note that we're not rooting the register result, if any; we depend + // on ResultsCollector::collect to root the value on our behalf, + // before causing any GC. + void* registerResultLoc = &exportArgs[0]; + DebugCodegen(DebugChannel::Function, "wasm-function[%d]; results [", + funcIndex); + if (!results.collect(cx, registerResultLoc, args.rval(), level)) { + return false; + } + DebugCodegen(DebugChannel::Function, "]\n"); + + return true; +} + +static JSObject* GetExceptionTag(JSObject* exn) { + return exn->is() ? &exn->as().tag() + : nullptr; +} + +void Instance::setPendingException(HandleAnyRef exn) { + pendingException_ = exn.get().asJSObject(); + pendingExceptionTag_ = GetExceptionTag(exn.get().asJSObject()); +} + +void Instance::constantGlobalGet(uint32_t globalIndex, + MutableHandleVal result) { + MOZ_RELEASE_ASSERT(globalIndex < maxInitializedGlobalsIndexPlus1_); + const GlobalDesc& global = metadata().globals[globalIndex]; + + // Constant globals are baked into the code and never stored in global data. + if (global.isConstant()) { + // We can just re-evaluate the global initializer to get the value. + result.set(Val(global.constantValue())); + return; + } + + // Otherwise, we need to load the initialized value from its cell. + const void* cell = addressOfGlobalCell(global); + result.address()->initFromHeapLocation(global.type(), cell); +} + +bool Instance::constantRefFunc(uint32_t funcIndex, + MutableHandleFuncRef result) { + void* fnref = Instance::refFunc(this, funcIndex); + if (fnref == AnyRef::invalid().forCompiledCode()) { + return false; // OOM, which has already been reported. + } + result.set(FuncRef::fromCompiledCode(fnref)); + return true; +} + +WasmStructObject* Instance::constantStructNewDefault(JSContext* cx, + uint32_t typeIndex) { + TypeDefInstanceData* typeDefData = typeDefInstanceData(typeIndex); + // We assume that constant structs will have a long lifetime and hence + // allocate them directly in the tenured heap. + return WasmStructObject::createStruct(cx, typeDefData, gc::Heap::Tenured); +} + +WasmArrayObject* Instance::constantArrayNewDefault(JSContext* cx, + uint32_t typeIndex, + uint32_t numElements) { + TypeDefInstanceData* typeDefData = typeDefInstanceData(typeIndex); + // We assume that constant arrays will have a long lifetime and hence + // allocate them directly in the tenured heap. + return WasmArrayObject::createArray(cx, typeDefData, gc::Heap::Tenured, + numElements); +} + +JSAtom* Instance::getFuncDisplayAtom(JSContext* cx, uint32_t funcIndex) const { + // The "display name" of a function is primarily shown in Error.stack which + // also includes location, so use getFuncNameBeforeLocation. + UTF8Bytes name; + if (!metadata().getFuncNameBeforeLocation(funcIndex, &name)) { + return nullptr; + } + + return AtomizeUTF8Chars(cx, name.begin(), name.length()); +} + +void Instance::ensureProfilingLabels(bool profilingEnabled) const { + return code_->ensureProfilingLabels(profilingEnabled); +} + +void Instance::onMovingGrowMemory() { + MOZ_ASSERT(!isAsmJS()); + MOZ_ASSERT(!memory_->isShared()); + + ArrayBufferObject& buffer = memory_->buffer().as(); + memoryBase_ = buffer.dataPointer(); + size_t limit = memory_->boundsCheckLimit(); +#if !defined(JS_64BIT) + // We assume that the limit is a 32-bit quantity + MOZ_ASSERT(limit <= UINT32_MAX); +#endif + boundsCheckLimit_ = limit; +} + +void Instance::onMovingGrowTable(const Table* theTable) { + MOZ_ASSERT(!isAsmJS()); + + // `theTable` has grown and we must update cached data for it. Importantly, + // we can have cached those data in more than one location: we'll have + // cached them once for each time the table was imported into this instance. + // + // When an instance is registered as an observer of a table it is only + // registered once, regardless of how many times the table was imported. + // Thus when a table is grown, onMovingGrowTable() is only invoked once for + // the table. + // + // Ergo we must go through the entire list of tables in the instance here + // and check for the table in all the cached-data slots; we can't exit after + // the first hit. + + for (uint32_t i = 0; i < tables_.length(); i++) { + if (tables_[i] == theTable) { + TableInstanceData& table = tableInstanceData(i); + table.length = tables_[i]->length(); + table.elements = tables_[i]->instanceElements(); + } + } +} + +JSString* Instance::createDisplayURL(JSContext* cx) { + // In the best case, we simply have a URL, from a streaming compilation of a + // fetched Response. + + if (metadata().filenameIsURL) { + const char* filename = metadata().filename.get(); + return NewStringCopyUTF8N(cx, JS::UTF8Chars(filename, strlen(filename))); + } + + // Otherwise, build wasm module URL from following parts: + // - "wasm:" as protocol; + // - URI encoded filename from metadata (if can be encoded), plus ":"; + // - 64-bit hash of the module bytes (as hex dump). + + JSStringBuilder result(cx); + if (!result.append("wasm:")) { + return nullptr; + } + + if (const char* filename = metadata().filename.get()) { + // EncodeURI returns false due to invalid chars or OOM -- fail only + // during OOM. + JSString* filenamePrefix = EncodeURI(cx, filename, strlen(filename)); + if (!filenamePrefix) { + if (cx->isThrowingOutOfMemory()) { + return nullptr; + } + + MOZ_ASSERT(!cx->isThrowingOverRecursed()); + cx->clearPendingException(); + return nullptr; + } + + if (!result.append(filenamePrefix)) { + return nullptr; + } + } + + if (metadata().debugEnabled) { + if (!result.append(":")) { + return nullptr; + } + + const ModuleHash& hash = metadata().debugHash; + for (unsigned char byte : hash) { + unsigned char digit1 = byte / 16, digit2 = byte % 16; + if (!result.append( + (char)(digit1 < 10 ? digit1 + '0' : digit1 + 'a' - 10))) { + return nullptr; + } + if (!result.append( + (char)(digit2 < 10 ? digit2 + '0' : digit2 + 'a' - 10))) { + return nullptr; + } + } + } + + return result.finishString(); +} + +WasmBreakpointSite* Instance::getOrCreateBreakpointSite(JSContext* cx, + uint32_t offset) { + MOZ_ASSERT(debugEnabled()); + return debug().getOrCreateBreakpointSite(cx, this, offset); +} + +void Instance::destroyBreakpointSite(JS::GCContext* gcx, uint32_t offset) { + MOZ_ASSERT(debugEnabled()); + return debug().destroyBreakpointSite(gcx, this, offset); +} + +void Instance::disassembleExport(JSContext* cx, uint32_t funcIndex, Tier tier, + PrintCallback printString) const { + const MetadataTier& metadataTier = metadata(tier); + const FuncExport& funcExport = metadataTier.lookupFuncExport(funcIndex); + const CodeRange& range = metadataTier.codeRange(funcExport); + const CodeTier& codeTier = code(tier); + const ModuleSegment& segment = codeTier.segment(); + + MOZ_ASSERT(range.begin() < segment.length()); + MOZ_ASSERT(range.end() < segment.length()); + + uint8_t* functionCode = segment.base() + range.begin(); + jit::Disassemble(functionCode, range.end() - range.begin(), printString); +} + +void Instance::addSizeOfMisc(MallocSizeOf mallocSizeOf, + Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, + Table::SeenSet* seenTables, size_t* code, + size_t* data) const { + *data += mallocSizeOf(this); + for (const SharedTable& table : tables_) { + *data += table->sizeOfIncludingThisIfNotSeen(mallocSizeOf, seenTables); + } + + if (maybeDebug_) { + maybeDebug_->addSizeOfMisc(mallocSizeOf, seenMetadata, seenCode, code, + data); + } + + code_->addSizeOfMiscIfNotSeen(mallocSizeOf, seenMetadata, seenCode, code, + data); +} + +////////////////////////////////////////////////////////////////////////////// +// +// Reporting of errors that are traps. + +void wasm::ReportTrapError(JSContext* cx, unsigned errorNumber) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, errorNumber); + + if (cx->isThrowingOutOfMemory()) { + return; + } + + // Mark the exception as thrown from a trap to prevent if from being handled + // by wasm exception handlers. + RootedValue exn(cx); + if (!cx->getPendingException(&exn)) { + return; + } + + MOZ_ASSERT(exn.isObject() && exn.toObject().is()); + exn.toObject().as().setFromWasmTrap(); +} diff --git a/js/src/wasm/WasmInstance.h b/js/src/wasm/WasmInstance.h new file mode 100644 index 0000000000..466eac73b1 --- /dev/null +++ b/js/src/wasm/WasmInstance.h @@ -0,0 +1,512 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_instance_h +#define wasm_instance_h + +#include "mozilla/Atomics.h" +#include "mozilla/Maybe.h" + +#include "gc/Barrier.h" +#include "gc/Zone.h" +#include "js/Stack.h" // JS::NativeStackLimit +#include "js/TypeDecls.h" +#include "vm/SharedMem.h" +#include "wasm/WasmExprType.h" // for ResultType +#include "wasm/WasmLog.h" // for PrintCallback +#include "wasm/WasmShareable.h" // for SeenSet +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmValue.h" + +namespace js { + +class SharedArrayRawBuffer; +class WasmBreakpointSite; + +class WasmStructObject; +class WasmArrayObject; + +namespace gc { +class StoreBuffer; +} // namespace gc + +namespace wasm { + +using mozilla::Atomic; + +class FuncImport; +struct FuncImportInstanceData; +class GlobalDesc; +struct TableDesc; +struct TableInstanceData; +struct TagDesc; +struct TagInstanceData; +struct TypeDefInstanceData; +class WasmFrameIter; + +// Instance represents a wasm instance and provides all the support for runtime +// execution of code in the instance. Instances share various immutable data +// structures with the Module from which they were instantiated and other +// instances instantiated from the same Module. However, an Instance has no +// direct reference to its source Module which allows a Module to be destroyed +// while it still has live Instances. +// +// The instance's code may be shared among multiple instances. +// +// An Instance is also known as a 'TlsData'. They used to be separate objects, +// but have now been unified. Extant references to 'TlsData' will be cleaned +// up over time. +class alignas(16) Instance { + // NOTE: The first fields of Instance are reserved for commonly accessed data + // from the JIT, such that they have as small an offset as possible. See the + // next note for the end of this region. + + // Pointer to the base of the default memory (or null if there is none). + uint8_t* memoryBase_; + + // Bounds check limit in bytes (or zero if there is no memory). This is + // 64-bits on 64-bit systems so as to allow for heap lengths up to and beyond + // 4GB, and 32-bits on 32-bit systems, where heaps are limited to 2GB. + // + // See "Linear memory addresses and bounds checking" in WasmMemory.cpp. + uintptr_t boundsCheckLimit_; + + // Null or a pointer to a per-process builtin thunk that will invoke the Debug + // Trap Handler. + void* debugTrapHandler_; + + // The containing JS::Realm. + JS::Realm* realm_; + + // The containing JSContext. + JSContext* cx_; + + // The pending exception that was found during stack unwinding after a throw. + // + // - Only non-null while unwinding the control stack from a wasm-exit stub. + // until the nearest enclosing Wasm try-catch or try-delegate block. + // - Set by wasm::HandleThrow, unset by Instance::consumePendingException. + // - If the unwind target is a `try-delegate`, it is unset by the delegated + // try-catch block or function body block. + GCPtr pendingException_; + // The tag object of the pending exception. + GCPtr pendingExceptionTag_; + + // Usually equal to cx->stackLimitForJitCode(JS::StackForUntrustedScript), + // but can be racily set to trigger immediate trap as an opportunity to + // CheckForInterrupt without an additional branch. + Atomic stackLimit_; + + // Set to 1 when wasm should call CheckForInterrupt. + Atomic interrupt_; + + // The address of the realm()->zone()->needsIncrementalBarrier(). This is + // specific to this instance and not a process wide field, and so it cannot + // be linked into code. + const JS::shadow::Zone::BarrierState* addressOfNeedsIncrementalBarrier_; + + public: + // NOTE: All fields commonly accessed by the JIT must be above this method, + // and this method adapted for the last field present. This method is used + // to assert that we can use compact offsets on x86(-64) for these fields. + // We cannot have the assertion here, due to C++ 'offsetof' rules. + static constexpr size_t offsetOfLastCommonJitField() { + return offsetof(Instance, addressOfNeedsIncrementalBarrier_); + } + + private: + // When compiling with tiering, the jumpTable has one entry for each + // baseline-compiled function. + void** jumpTable_; + + // General scratch storage for the baseline compiler, which can't always use + // the stack for this. + uint32_t baselineScratch_[2]; + + // The class_ of WasmValueBox, this is a per-process value. We could patch + // this into code, but the only use-sites are register restricted and cannot + // easily use a symbolic address. + const JSClass* valueBoxClass_; + + // Address of the JitRuntime's arguments rectifier trampoline + void* jsJitArgsRectifier_; + + // Address of the JitRuntime's exception handler trampoline + void* jsJitExceptionHandler_; + + // Address of the JitRuntime's object prebarrier trampoline + void* preBarrierCode_; + + // Address of the store buffer for this instance + gc::StoreBuffer* storeBuffer_; + + // Weak pointer to WasmInstanceObject that owns this instance + WeakHeapPtr object_; + + // The wasm::Code for this instance + const SharedCode code_; + + // The memory for this instance, if any + const GCPtr memory_; + + // The tables for this instance, if any + const SharedTableVector tables_; + + // Passive data segments for use with bulk memory instructions + DataSegmentVector passiveDataSegments_; + + // Passive elem segments for use with bulk memory instructions + ElemSegmentVector passiveElemSegments_; + + // The wasm::DebugState for this instance, if any + const UniqueDebugState maybeDebug_; + + // If debugging, this is a per-funcIndex bit table denoting whether debugging + // is currently enabled for the function within the instance. The flag is set + // if any breakpoint or function entry or exit point needs to be visited. It + // is OK to conservatively set this flag, but there is very significant + // overhead to taking a breakpoint trap, so managing it precisely is + // worthwhile. + uint32_t* debugFilter_; + + // The exclusive maximum index of a global that has been initialized so far. + uint32_t maxInitializedGlobalsIndexPlus1_; + + // Pointer that should be freed (due to padding before the Instance). + void* allocatedBase_; + + // The data must be the last field. Globals for the module start here + // and are inline in this structure. 16-byte alignment is required for SIMD + // data. + MOZ_ALIGNED_DECL(16, char data_); + + // Internal helpers: + TypeDefInstanceData* typeDefInstanceData(uint32_t typeIndex) const; + const void* addressOfGlobalCell(const GlobalDesc& globalDesc) const; + FuncImportInstanceData& funcImportInstanceData(const FuncImport& fi); + TableInstanceData& tableInstanceData(uint32_t tableIndex) const; + TagInstanceData& tagInstanceData(uint32_t tagIndex) const; + + // Only WasmInstanceObject can call the private trace function. + friend class js::WasmInstanceObject; + void tracePrivate(JSTracer* trc); + + bool callImport(JSContext* cx, uint32_t funcImportIndex, unsigned argc, + uint64_t* argv); + + Instance(JSContext* cx, Handle object, + const SharedCode& code, Handle memory, + SharedTableVector&& tables, UniqueDebugState maybeDebug); + ~Instance(); + + public: + static Instance* create(JSContext* cx, Handle object, + const SharedCode& code, uint32_t instanceDataLength, + Handle memory, + SharedTableVector&& tables, + UniqueDebugState maybeDebug); + static void destroy(Instance* instance); + + bool init(JSContext* cx, const JSObjectVector& funcImports, + const ValVector& globalImportValues, + const WasmGlobalObjectVector& globalObjs, + const WasmTagObjectVector& tagObjs, + const DataSegmentVector& dataSegments, + const ElemSegmentVector& elemSegments); + + // Trace any GC roots on the stack, for the frame associated with |wfi|, + // whose next instruction to execute is |nextPC|. + // + // For consistency checking of StackMap sizes in debug builds, this also + // takes |highestByteVisitedInPrevFrame|, which is the address of the + // highest byte scanned in the frame below this one on the stack, and in + // turn it returns the address of the highest byte scanned in this frame. + uintptr_t traceFrame(JSTracer* trc, const wasm::WasmFrameIter& wfi, + uint8_t* nextPC, + uintptr_t highestByteVisitedInPrevFrame); + + static constexpr size_t offsetOfMemoryBase() { + return offsetof(Instance, memoryBase_); + } + static constexpr size_t offsetOfBoundsCheckLimit() { + return offsetof(Instance, boundsCheckLimit_); + } + static constexpr size_t offsetOfDebugTrapHandler() { + return offsetof(Instance, debugTrapHandler_); + } + + static constexpr size_t offsetOfRealm() { return offsetof(Instance, realm_); } + static constexpr size_t offsetOfCx() { return offsetof(Instance, cx_); } + static constexpr size_t offsetOfValueBoxClass() { + return offsetof(Instance, valueBoxClass_); + } + static constexpr size_t offsetOfPendingException() { + return offsetof(Instance, pendingException_); + } + static constexpr size_t offsetOfPendingExceptionTag() { + return offsetof(Instance, pendingExceptionTag_); + } + static constexpr size_t offsetOfStackLimit() { + return offsetof(Instance, stackLimit_); + } + static constexpr size_t offsetOfInterrupt() { + return offsetof(Instance, interrupt_); + } + static constexpr size_t offsetOfAddressOfNeedsIncrementalBarrier() { + return offsetof(Instance, addressOfNeedsIncrementalBarrier_); + } + static constexpr size_t offsetOfJumpTable() { + return offsetof(Instance, jumpTable_); + } + static constexpr size_t offsetOfBaselineScratch() { + return offsetof(Instance, baselineScratch_); + } + static constexpr size_t sizeOfBaselineScratch() { + return sizeof(baselineScratch_); + } + static constexpr size_t offsetOfJSJitArgsRectifier() { + return offsetof(Instance, jsJitArgsRectifier_); + } + static constexpr size_t offsetOfJSJitExceptionHandler() { + return offsetof(Instance, jsJitExceptionHandler_); + } + static constexpr size_t offsetOfPreBarrierCode() { + return offsetof(Instance, preBarrierCode_); + } + static constexpr size_t offsetOfDebugFilter() { + return offsetof(Instance, debugFilter_); + } + static constexpr size_t offsetOfData() { return offsetof(Instance, data_); } + static constexpr size_t offsetInData(size_t offset) { + return offsetOfData() + offset; + } + + JSContext* cx() const { return cx_; } + void* debugTrapHandler() const { return debugTrapHandler_; } + void setDebugTrapHandler(void* newHandler) { debugTrapHandler_ = newHandler; } + JS::Realm* realm() const { return realm_; } + bool debugEnabled() const { return !!maybeDebug_; } + DebugState& debug() { return *maybeDebug_; } + uint8_t* data() const { return (uint8_t*)&data_; } + const SharedTableVector& tables() const { return tables_; } + SharedMem memoryBase() const; + WasmMemoryObject* memory() const; + size_t memoryMappedSize() const; + SharedArrayRawBuffer* sharedMemoryBuffer() const; // never null + bool memoryAccessInGuardRegion(const uint8_t* addr, unsigned numBytes) const; + + // Methods to set, test and clear the interrupt fields. Both interrupt + // fields are Relaxed and so no consistency/ordering can be assumed. + + void setInterrupt(); + bool isInterrupted() const; + void resetInterrupt(JSContext* cx); + + bool debugFilter(uint32_t funcIndex) const; + void setDebugFilter(uint32_t funcIndex, bool value); + + const Code& code() const { return *code_; } + inline const CodeTier& code(Tier t) const; + inline uint8_t* codeBase(Tier t) const; + inline const MetadataTier& metadata(Tier t) const; + inline const Metadata& metadata() const; + inline bool isAsmJS() const; + + // This method returns a pointer to the GC object that owns this Instance. + // Instances may be reached via weak edges (e.g., Realm::instances_) + // so this perform a read-barrier on the returned object unless the barrier + // is explicitly waived. + + WasmInstanceObject* object() const; + WasmInstanceObject* objectUnbarriered() const; + + // Execute the given export given the JS call arguments, storing the return + // value in args.rval. + + [[nodiscard]] bool callExport(JSContext* cx, uint32_t funcIndex, + CallArgs args, + CoercionLevel level = CoercionLevel::Spec); + + // Exception handling support + + void setPendingException(HandleAnyRef exn); + + // Constant expression support + + void constantGlobalGet(uint32_t globalIndex, MutableHandleVal result); + [[nodiscard]] bool constantRefFunc(uint32_t funcIndex, + MutableHandleFuncRef result); + WasmStructObject* constantStructNewDefault(JSContext* cx, uint32_t typeIndex); + WasmArrayObject* constantArrayNewDefault(JSContext* cx, uint32_t typeIndex, + uint32_t numElements); + + // Return the name associated with a given function index, or generate one + // if none was given by the module. + + JSAtom* getFuncDisplayAtom(JSContext* cx, uint32_t funcIndex) const; + void ensureProfilingLabels(bool profilingEnabled) const; + + // Called by Wasm(Memory|Table)Object when a moving resize occurs: + + void onMovingGrowMemory(); + void onMovingGrowTable(const Table* theTable); + + // Called to apply a single ElemSegment at a given offset, assuming + // that all bounds validation has already been performed. + + [[nodiscard]] bool initElems(uint32_t tableIndex, const ElemSegment& seg, + uint32_t dstOffset, uint32_t srcOffset, + uint32_t len); + + // Debugger support: + + JSString* createDisplayURL(JSContext* cx); + WasmBreakpointSite* getOrCreateBreakpointSite(JSContext* cx, uint32_t offset); + void destroyBreakpointSite(JS::GCContext* gcx, uint32_t offset); + + // about:memory reporting: + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, SeenSet* seenMetadata, + SeenSet* seenCode, SeenSet
* seenTables, + size_t* code, size_t* data) const; + + // Wasm disassembly support + + void disassembleExport(JSContext* cx, uint32_t funcIndex, Tier tier, + PrintCallback printString) const; + + public: + // Functions to be called directly from wasm code. + static int32_t callImport_general(Instance*, int32_t, int32_t, uint64_t*); + static uint32_t memoryGrow_m32(Instance* instance, uint32_t delta); + static uint64_t memoryGrow_m64(Instance* instance, uint64_t delta); + static uint32_t memorySize_m32(Instance* instance); + static uint64_t memorySize_m64(Instance* instance); + static int32_t memCopy_m32(Instance* instance, uint32_t dstByteOffset, + uint32_t srcByteOffset, uint32_t len, + uint8_t* memBase); + static int32_t memCopyShared_m32(Instance* instance, uint32_t dstByteOffset, + uint32_t srcByteOffset, uint32_t len, + uint8_t* memBase); + static int32_t memCopy_m64(Instance* instance, uint64_t dstByteOffset, + uint64_t srcByteOffset, uint64_t len, + uint8_t* memBase); + static int32_t memCopyShared_m64(Instance* instance, uint64_t dstByteOffset, + uint64_t srcByteOffset, uint64_t len, + uint8_t* memBase); + static int32_t memFill_m32(Instance* instance, uint32_t byteOffset, + uint32_t value, uint32_t len, uint8_t* memBase); + static int32_t memFillShared_m32(Instance* instance, uint32_t byteOffset, + uint32_t value, uint32_t len, + uint8_t* memBase); + static int32_t memFill_m64(Instance* instance, uint64_t byteOffset, + uint32_t value, uint64_t len, uint8_t* memBase); + static int32_t memFillShared_m64(Instance* instance, uint64_t byteOffset, + uint32_t value, uint64_t len, + uint8_t* memBase); + static int32_t memInit_m32(Instance* instance, uint32_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t segIndex); + static int32_t memInit_m64(Instance* instance, uint64_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t segIndex); + static int32_t dataDrop(Instance* instance, uint32_t segIndex); + static int32_t tableCopy(Instance* instance, uint32_t dstOffset, + uint32_t srcOffset, uint32_t len, + uint32_t dstTableIndex, uint32_t srcTableIndex); + static int32_t tableFill(Instance* instance, uint32_t start, void* value, + uint32_t len, uint32_t tableIndex); + static int32_t memDiscard_m32(Instance* instance, uint32_t byteOffset, + uint32_t byteLen, uint8_t* memBase); + static int32_t memDiscardShared_m32(Instance* instance, uint32_t byteOffset, + uint32_t byteLen, uint8_t* memBase); + static int32_t memDiscard_m64(Instance* instance, uint64_t byteOffset, + uint64_t byteLen, uint8_t* memBase); + static int32_t memDiscardShared_m64(Instance* instance, uint64_t byteOffset, + uint64_t byteLen, uint8_t* memBase); + static void* tableGet(Instance* instance, uint32_t index, + uint32_t tableIndex); + static uint32_t tableGrow(Instance* instance, void* initValue, uint32_t delta, + uint32_t tableIndex); + static int32_t tableSet(Instance* instance, uint32_t index, void* value, + uint32_t tableIndex); + static uint32_t tableSize(Instance* instance, uint32_t tableIndex); + static int32_t tableInit(Instance* instance, uint32_t dstOffset, + uint32_t srcOffset, uint32_t len, uint32_t segIndex, + uint32_t tableIndex); + static int32_t elemDrop(Instance* instance, uint32_t segIndex); + static int32_t wait_i32_m32(Instance* instance, uint32_t byteOffset, + int32_t value, int64_t timeout); + static int32_t wait_i32_m64(Instance* instance, uint64_t byteOffset, + int32_t value, int64_t timeout); + static int32_t wait_i64_m32(Instance* instance, uint32_t byteOffset, + int64_t value, int64_t timeout); + static int32_t wait_i64_m64(Instance* instance, uint64_t byteOffset, + int64_t value, int64_t timeout); + static int32_t wake_m32(Instance* instance, uint32_t byteOffset, + int32_t count); + static int32_t wake_m64(Instance* instance, uint64_t byteOffset, + int32_t count); + static void* refFunc(Instance* instance, uint32_t funcIndex); + static void postBarrier(Instance* instance, gc::Cell** location); + static void postBarrierPrecise(Instance* instance, JSObject** location, + JSObject* prev); + static void postBarrierPreciseWithOffset(Instance* instance, JSObject** base, + uint32_t offset, JSObject* prev); + static void* exceptionNew(Instance* instance, JSObject* tag); + static int32_t throwException(Instance* instance, JSObject* exn); + static void* structNew(Instance* instance, TypeDefInstanceData* typeDefData); + static void* structNewUninit(Instance* instance, + TypeDefInstanceData* typeDefData); + static void* arrayNew(Instance* instance, uint32_t numElements, + TypeDefInstanceData* typeDefData); + static void* arrayNewUninit(Instance* instance, uint32_t numElements, + TypeDefInstanceData* typeDefData); + static void* arrayNewData(Instance* instance, uint32_t segByteOffset, + uint32_t numElements, + TypeDefInstanceData* typeDefData, + uint32_t segIndex); + static void* arrayNewElem(Instance* instance, uint32_t segElemIndex, + uint32_t numElements, + TypeDefInstanceData* typeDefData, + uint32_t segIndex); + static int32_t arrayCopy(Instance* instance, void* dstArray, + uint32_t dstIndex, void* srcArray, uint32_t srcIndex, + uint32_t numElements, uint32_t elementSize); + static int32_t refTest(Instance* instance, void* refPtr, + const wasm::TypeDef* typeDef); + static int32_t intrI8VecMul(Instance* instance, uint32_t dest, uint32_t src1, + uint32_t src2, uint32_t len, uint8_t* memBase); +}; + +bool ResultsToJSValue(JSContext* cx, ResultType type, void* registerResultLoc, + Maybe stackResultsLoc, MutableHandleValue rval, + CoercionLevel level = CoercionLevel::Spec); + +// Report an error to `cx` and mark it as a 'trap' so that it cannot be caught +// by wasm exception handlers. +void ReportTrapError(JSContext* cx, unsigned errorNumber); + +// Instance is not a GC thing itself but contains GC thing pointers. Ensure they +// are traced appropriately. +void TraceInstanceEdge(JSTracer* trc, Instance* instance, const char* name); + +} // namespace wasm +} // namespace js + +#endif // wasm_instance_h diff --git a/js/src/wasm/WasmInstanceData.h b/js/src/wasm/WasmInstanceData.h new file mode 100644 index 0000000000..b1be3f02a2 --- /dev/null +++ b/js/src/wasm/WasmInstanceData.h @@ -0,0 +1,137 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_instance_data_h +#define wasm_instance_data_h + +#include + +#include "NamespaceImports.h" + +#include "gc/Allocator.h" +#include "gc/Pretenuring.h" +#include "js/Utility.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmTypeDecls.h" + +namespace js { +namespace wasm { + +// ExportArg holds the unboxed operands to the wasm entry trampoline which can +// be called through an ExportFuncPtr. + +struct ExportArg { + uint64_t lo; + uint64_t hi; +}; + +using ExportFuncPtr = int32_t (*)(ExportArg*, Instance*); + +// TypeDefInstanceData describes the runtime information associated with a +// module's type definition. This is accessed directly from JIT code and the +// Instance. + +struct TypeDefInstanceData { + TypeDefInstanceData() + : typeDef(nullptr), + superTypeVector(nullptr), + shape(nullptr), + clasp(nullptr), + allocKind(gc::AllocKind::LIMIT) {} + + // The canonicalized pointer to this type definition. This is kept alive by + // the type context associated with the instance. + const wasm::TypeDef* typeDef; + + // The supertype vector for this type definition. This is also kept alive + // by the type context associated with the instance. + // + const wasm::SuperTypeVector* superTypeVector; + + // The remaining fields are only meaningful for, and used by, structs and + // arrays. + GCPtr shape; + const JSClass* clasp; + // The allocation site for GC types. This is used for pre-tenuring. + gc::AllocSite allocSite; + gc::AllocKind allocKind; +}; + +// FuncImportInstanceData describes the region of wasm global memory allocated +// in the instance's thread-local storage for a function import. This is +// accessed directly from JIT code and mutated by Instance as exits become +// optimized and deoptimized. + +struct FuncImportInstanceData { + // The code to call at an import site: a wasm callee, a thunk into C++, or a + // thunk into JIT code. + void* code; + + // The callee's Instance pointer, which must be loaded to InstanceReg + // (along with any pinned registers) before calling 'code'. + Instance* instance; + + // The callee function's realm. + JS::Realm* realm; + + // A GC pointer which keeps the callee alive and is used to recover import + // values for lazy table initialization. + GCPtr callable; + static_assert(sizeof(GCPtr) == sizeof(void*), "for JIT access"); +}; + +// TableInstanceData describes the region of wasm global memory allocated in the +// instance's thread-local storage which is accessed directly from JIT code +// to bounds-check and index the table. + +struct TableInstanceData { + // Length of the table in number of elements (not bytes). + uint32_t length; + + // Pointer to the array of elements (which can have various representations). + // For tables of anyref this is null. + // For tables of functions, this is a pointer to the array of code pointers. + void* elements; +}; + +// TagInstanceData describes the instance state associated with a tag. + +struct TagInstanceData { + GCPtr object; +}; + +// Table element for TableRepr::Func which carries both the code pointer and +// a instance pointer (and thus anything reachable through the instance). + +struct FunctionTableElem { + // The code to call when calling this element. The table ABI is the system + // ABI with the additional ABI requirements that: + // - InstanceReg and any pinned registers have been loaded appropriately + // - if this is a heterogeneous table that requires a signature check, + // WasmTableCallSigReg holds the signature id. + void* code; + + // The pointer to the callee's instance's Instance. This must be loaded into + // InstanceReg before calling 'code'. + Instance* instance; +}; + +} // namespace wasm +} // namespace js + +#endif // wasm_instance_data_h diff --git a/js/src/wasm/WasmIntrinsic.cpp b/js/src/wasm/WasmIntrinsic.cpp new file mode 100644 index 0000000000..cc43ed887c --- /dev/null +++ b/js/src/wasm/WasmIntrinsic.cpp @@ -0,0 +1,239 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmIntrinsic.h" + +#include "util/Text.h" +#include "vm/GlobalObject.h" + +#include "wasm/WasmGenerator.h" +#include "wasm/WasmIntrinsicGenerated.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmModule.h" +#include "wasm/WasmOpIter.h" +#include "wasm/WasmValidate.h" + +using namespace js; +using namespace js::wasm; + +#define INTR_DECL(op, export, sa_name, abitype, entry, idx) \ + static const ValType Intrinsic##op##_Params[] = \ + DECLARE_INTRINSIC_SAS_PARAM_VALTYPES_##op; \ + \ + const Intrinsic Intrinsic##op = { \ + export, \ + mozilla::Span(Intrinsic##op##_Params), \ + SASig##sa_name, \ + }; + +FOR_EACH_INTRINSIC(INTR_DECL) +#undef INTR_DECL + +bool Intrinsic::funcType(FuncType* type) const { + ValTypeVector paramVec; + if (!paramVec.append(params.data(), params.data() + params.size())) { + return false; + } + *type = FuncType(std::move(paramVec), ValTypeVector()); + return true; +} + +/* static */ +const Intrinsic& Intrinsic::getFromId(IntrinsicId id) { + switch (id) { +#define OP(op, export, sa_name, abitype, entry, idx) \ + case IntrinsicId::op: \ + return Intrinsic##op; + FOR_EACH_INTRINSIC(OP) +#undef OP + default: + MOZ_CRASH("unexpected intrinsic"); + } +} + +bool EncodeIntrinsicBody(const Intrinsic& intrinsic, IntrinsicId id, + Bytes* body) { + Encoder encoder(*body); + if (!EncodeLocalEntries(encoder, ValTypeVector())) { + return false; + } + for (uint32_t i = 0; i < intrinsic.params.size(); i++) { + if (!encoder.writeOp(Op::LocalGet) || !encoder.writeVarU32(i)) { + return false; + } + } + if (!encoder.writeOp(MozOp::Intrinsic)) { + return false; + } + if (!encoder.writeVarU32(uint32_t(id))) { + return false; + } + if (!encoder.writeOp(Op::End)) { + return false; + } + return true; +} + +bool wasm::CompileIntrinsicModule(JSContext* cx, + const mozilla::Span ids, + Shareable sharedMemory, + MutableHandle result) { + // Create the options manually, enabling intrinsics + FeatureOptions featureOptions; + featureOptions.intrinsics = true; + + // Initialize the compiler environment, choosing the best tier possible + SharedCompileArgs compileArgs = CompileArgs::buildAndReport( + cx, ScriptedCaller(), featureOptions, /* reportOOM */ true); + if (!compileArgs) { + return false; + } + CompilerEnvironment compilerEnv( + CompileMode::Once, IonAvailable(cx) ? Tier::Optimized : Tier::Baseline, + DebugEnabled::False); + compilerEnv.computeParameters(); + + // Build a module environment + ModuleEnvironment moduleEnv(compileArgs->features); + if (!moduleEnv.init()) { + ReportOutOfMemory(cx); + return false; + } + + // Add (import (memory 0)) + CacheableName emptyString; + CacheableName memoryString; + if (!CacheableName::fromUTF8Chars("memory", &memoryString)) { + ReportOutOfMemory(cx); + return false; + } + if (!moduleEnv.imports.append(Import(std::move(emptyString), + std::move(memoryString), + DefinitionKind::Memory))) { + ReportOutOfMemory(cx); + return false; + } + moduleEnv.memory = Some(MemoryDesc(Limits(0, Nothing(), sharedMemory))); + + // Add (type (func (params ...))) for each intrinsic. The function types will + // be deduplicated by the runtime + for (uint32_t funcIndex = 0; funcIndex < ids.size(); funcIndex++) { + const IntrinsicId& id = ids[funcIndex]; + const Intrinsic& intrinsic = Intrinsic::getFromId(id); + + FuncType type; + if (!intrinsic.funcType(&type) || + !moduleEnv.types->addType(std::move(type))) { + ReportOutOfMemory(cx); + return false; + } + } + + // Add (func (type $i)) declarations. Do this after all types have been added + // as the function declaration metadata uses pointers into the type vectors + // that must be stable. + for (uint32_t funcIndex = 0; funcIndex < ids.size(); funcIndex++) { + FuncDesc decl(&(*moduleEnv.types)[funcIndex].funcType(), funcIndex); + if (!moduleEnv.funcs.append(decl)) { + ReportOutOfMemory(cx); + return false; + } + moduleEnv.declareFuncExported(funcIndex, true, false); + } + + // Add (export "$name" (func $i)) declarations. + for (uint32_t funcIndex = 0; funcIndex < ids.size(); funcIndex++) { + const Intrinsic& intrinsic = Intrinsic::getFromId(ids[funcIndex]); + + CacheableName exportName; + if (!CacheableName::fromUTF8Chars(intrinsic.exportName, &exportName) || + !moduleEnv.exports.append(Export(std::move(exportName), funcIndex, + DefinitionKind::Function))) { + ReportOutOfMemory(cx); + return false; + } + } + + // Compile the module functions + UniqueChars error; + ModuleGenerator mg(*compileArgs, &moduleEnv, &compilerEnv, nullptr, &error, + nullptr); + if (!mg.init(nullptr)) { + ReportOutOfMemory(cx); + return false; + } + + // Prepare and compile function bodies + Vector bodies; + if (!bodies.reserve(ids.size())) { + ReportOutOfMemory(cx); + return false; + } + for (uint32_t funcIndex = 0; funcIndex < ids.size(); funcIndex++) { + IntrinsicId id = ids[funcIndex]; + const Intrinsic& intrinsic = Intrinsic::getFromId(ids[funcIndex]); + + // Compilation may be done using other threads, ModuleGenerator requires + // that function bodies live until after finishFuncDefs(). + bodies.infallibleAppend(Bytes()); + Bytes& bytecode = bodies.back(); + + // Encode function body that will call the intrinsic using our builtin + // opcode, and launch a compile task + if (!EncodeIntrinsicBody(intrinsic, id, &bytecode) || + !mg.compileFuncDef(funcIndex, 0, bytecode.begin(), + bytecode.begin() + bytecode.length())) { + // This must be an OOM and will be reported by the caller + MOZ_ASSERT(!error); + ReportOutOfMemory(cx); + return false; + } + } + + // Finish and block on function compilation + if (!mg.finishFuncDefs()) { + // This must be an OOM and will be reported by the caller + MOZ_ASSERT(!error); + ReportOutOfMemory(cx); + return false; + } + + // Create a dummy bytecode vector, that will not be used + SharedBytes bytecode = js_new(); + if (!bytecode) { + ReportOutOfMemory(cx); + return false; + } + + // Finish the module + SharedModule module = mg.finishModule(*bytecode, nullptr); + if (!module) { + ReportOutOfMemory(cx); + return false; + } + + // Create a WasmModuleObject for the module, and return it + RootedObject proto( + cx, GlobalObject::getOrCreatePrototype(cx, JSProto_WasmModule)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + result.set(WasmModuleObject::create(cx, *module, proto)); + return !!result; +} diff --git a/js/src/wasm/WasmIntrinsic.h b/js/src/wasm/WasmIntrinsic.h new file mode 100644 index 0000000000..ff4bc2d002 --- /dev/null +++ b/js/src/wasm/WasmIntrinsic.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_intrinsic_h +#define wasm_intrinsic_h + +#include "mozilla/Span.h" + +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCompileArgs.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmTypeDef.h" + +namespace js { +namespace wasm { + +// An intrinsic is a natively implemented function that may be compiled into an +// 'intrinsic module', which may be instantiated with a provided memory +// yielding an exported WebAssembly function wrapping the intrinsic. +struct Intrinsic { + // The name of the intrinsic as it is exported + const char* exportName; + // The params taken by the intrinsic. No results are required for intrinsics + // at this time, so we omit them + mozilla::Span params; + // The signature of the builtin that implements the intrinsic + const SymbolicAddressSignature& signature; + + // Allocate a FuncType for this intrinsic, returning false for OOM + bool funcType(FuncType* type) const; + + // Get the Intrinsic for an IntrinsicId. IntrinsicId must be validated. + static const Intrinsic& getFromId(IntrinsicId id); +}; + +// Compile and return the intrinsic module for a given set of operations. +bool CompileIntrinsicModule(JSContext* cx, const mozilla::Span ids, + Shareable sharedMemory, + MutableHandle result); + +} // namespace wasm +} // namespace js + +#endif // wasm_intrinsic_h diff --git a/js/src/wasm/WasmIntrinsic.yaml b/js/src/wasm/WasmIntrinsic.yaml new file mode 100644 index 0000000000..7867604915 --- /dev/null +++ b/js/src/wasm/WasmIntrinsic.yaml @@ -0,0 +1,201 @@ +# The file contains information need to define wasm intrinsic operations. + +# i8vecmul(dest: i32, src1: i32, src2: i32, len: i32) +# Performs pairwise multiplication of two i8 vectors of 'len' specified at +# 'src1' and 'src2'. Output is written to 'dest'. This is used as a +# basic self-test for intrinsics. +- op: I8VecMul + symbolic_address: + name: IntrI8VecMul + type: Args_Int32_GeneralInt32Int32Int32Int32General + entry: Instance::intrI8VecMul + export: i8vecmul + params: + - I32 + - I32 + - I32 + - I32 + +#if defined(ENABLE_WASM_MOZ_INTGEMM) + +# Intrinsics for integer matrix multiplication followed by addition of bias. +# Please refer to @TOPSRCDIR/js/src/intgemm/IntegerGemmIntrinsic.h for more details on these intrinsics. + + +# Prepare B for the Matrix Multiply intrinsic from Input matrix B. +# +# Quantization is performed on the input. +# The final prepared B is in CPU-dependent format and can be used as an input to matrix multiply +# intrinsic (`int8_multiply_and_add_bias`). +# +# int8_prepare_b(const float* inputMatrixB, float scale, float zeroPoint, uint32_t rowsB, uint32_t colsB, int8_t* outputMatrixB) +# int8_prepare_b(inputMatrixB: i32, scale: f32, zeroPoint: f32, rowsB: i32, colsB: i32, outputMatrixB: i32) +- op: I8PrepareB + symbolic_address: + name: IntrI8PrepareB + type: Args_Int32_GeneralInt32Float32Float32Int32Int32Int32General + entry: intgemm::IntrI8PrepareB + export: int8_prepare_b + params: + - I32 + - F32 + - F32 + - I32 + - I32 + - I32 + + +# Prepare B for the Matrix Multiply intrinsic from transposed version of Input matrix B. +# +# Quantization is performed on floating values of input. +# The final prepared B is in CPU-dependent format and can be used as an input to matrix multiply +# intrinsic (`int8_multiply_and_add_bias`). +# +# int8_prepare_b_from_transposed(const float* inputMatrixBTransposed, float scale, float zeroPoint, uint32_t rowsB, uint32_t colsB, int8_t* outputMatrixB) +# int8_prepare_b_from_transposed(inputMatrixBTransposed: i32, scale: f32, zeroPoint: f32, rowsB: i32, colsB: i32, outputMatrixB: i32) +- op: I8PrepareBFromTransposed + symbolic_address: + name: IntrI8PrepareBFromTransposed + type: Args_Int32_GeneralInt32Float32Float32Int32Int32Int32General + entry: intgemm::IntrI8PrepareBFromTransposed + export: int8_prepare_b_from_transposed + params: + - I32 + - F32 + - F32 + - I32 + - I32 + - I32 + + +# Prepare B for the Matrix Multiply intrinsic from a quantized and transposed version of Input +# matrix B which is also in a CPU-independent format. +# +# The final prepared B is in CPU-dependent format and can be used as an input to matrix multiply +# intrinsic (`int8_multiply_and_add_bias`). +# +# int8_prepare_b_from_quantized_transposed(const int8_t* inputMatrixBQuantizedTransposed, uint32_t rowsB, uint32_t colsB, int8_t* outputMatrixB) +# int8_prepare_b_from_quantized_transposed(inputMatrixBQuantizedTransposed: i32, rowsB: i32, colsB: i32, outputMatrixB: i32) +- op: I8PrepareBFromQuantizedTransposed + symbolic_address: + name: IntrI8PrepareBFromQuantizedTransposed + type: Args_Int32_GeneralInt32Int32Int32Int32General + entry: intgemm::IntrI8PrepareBFromQuantizedTransposed + export: int8_prepare_b_from_quantized_transposed + params: + - I32 + - I32 + - I32 + - I32 + + +# Prepare A for the Matrix Multiply intrinsic from Input matrix A. +# +# It performs quantization on floating values of input. +# The final prepared A might be architecture dependent. e.g. On some architectures like x86, it +# might be unsigned (achieved by adding 127 to quantized values) while on others like Arm, it might +# be signed. +# The final prepared A can be used as an input to matrix multiply intrinsic +# (`int8_multiply_and_add_bias`). +# +# int8_prepare_a(const float* inputMatrixA, float scale, float zeroPoint, uint32_t rowsA, uint32_t colsA, int8_t* outputMatrixA) +# int8_prepare_a(inputMatrixA: i32, scale: f32, zeroPoint: f32, rowsA: i32, colsA: i32, outputMatrixA: i32) +- op: I8PrepareA + symbolic_address: + name: IntrI8PrepareA + type: Args_Int32_GeneralInt32Float32Float32Int32Int32Int32General + entry: intgemm::IntrI8PrepareA + export: int8_prepare_a + params: + - I32 + - F32 + - F32 + - I32 + - I32 + - I32 + + +# Prepares bias for the Matrix Multiply intrinsic. +# +# It uses the prepared B (which must be obtained by using any of the `int8_prepare_b*` intrinsics) and +# a bias input to prepare the final bias. +# +# The final bias can be used as an input to matrix multiply intrinsic (`int8_multiply_and_add_bias`). +# +# int8_prepare_bias(const int8_t* inputMatrixBPrepared, float scaleA, float zeroPointA, float scaleB, float zeroPointB, uint32_t rowsB, uint32_t colsB, const float* inputBias, float* output) +# int8_prepare_bias(inputMatrixBPrepared: i32, scaleA: f32, zeroPointA: f32, scaleB: f32, zeroPointB: f32, rowsB: i32, colsB: i32, inputBias: i32, output: i32) +- op: I8PrepareBias + symbolic_address: + name: IntrI8PrepareBias + type: Args_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General + entry: intgemm::IntrI8PrepareBias + export: int8_prepare_bias + params: + - I32 + - F32 + - F32 + - F32 + - F32 + - I32 + - I32 + - I32 + - I32 + + +# Perform multiplication of 2 matrices followed by adding a bias. +# +# i.e Output = inputMatrixAPrepared * inputMatrixBPrepared + inputBiasPrepared +# +# The inputs of this intrinsic must be obtained by using `int8_prepare_A`, +# one of the `int8_prepare_b*` and `int8_prepare_bias` intrinsics respectively. +# +# int8_multiply_and_add_bias(const int8_t* inputMatrixAPrepared, float scaleA, float zeroPointA, +# const int8_t* inputMatrixBPrepared, float scaleB, float zeroPointB, +# const float* inputBiasPrepared, float unquantMultiplier, +# uint32_t rowsA, uint32_t width, uint32_t colsB, float* output) +# int8_multiply_and_add_bias(inputMatrixAPrepared: i32, scaleA: f32, zeroPointA: f32, +# inputMatrixBPrepared: i32, scaleB: f32, zeroPointB: f32, +# inputBiasPrepared: i32, unquantMultiplier: f32, +# rowsA: i32, width: i32, colsB: i32, output: i32) +- op: I8MultiplyAndAddBias + symbolic_address: + name: IntrI8MultiplyAndAddBias + type: Args_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General + entry: intgemm::IntrI8MultiplyAndAddBias + export: int8_multiply_and_add_bias + params: + - I32 + - F32 + - F32 + - I32 + - F32 + - F32 + - I32 + - F32 + - I32 + - I32 + - I32 + - I32 + + +# Select a subset of columns of prepared B. +# +# Indices of the columns to be selected are specified by an array. +# +# int8_select_columns_of_b(const int8_t* inputMatrixBPrepared, uint32_t rowsB, uint32_t colsB, const uint32_t* colIndexList, const uint32_t sizeColIndexList, int8_t* output) +# int8_select_columns_of_b(inputMatrixBPrepared: i32, rowsB: i32, colsB: i32, colIndexList: i32, sizeColIndexList: i32, output: i32) +- op: I8SelectColumnsOfB + symbolic_address: + name: IntrI8SelectColumnsOfB + type: Args_Int32_GeneralInt32Int32Int32Int32Int32Int32General + entry: intgemm::IntrI8SelectColumnsOfB + export: int8_select_columns_of_b + params: + - I32 + - I32 + - I32 + - I32 + - I32 + - I32 + +#endif // ENABLE_WASM_MOZ_INTGEMM diff --git a/js/src/wasm/WasmIonCompile.cpp b/js/src/wasm/WasmIonCompile.cpp new file mode 100644 index 0000000000..2f5e3f7233 --- /dev/null +++ b/js/src/wasm/WasmIonCompile.cpp @@ -0,0 +1,8691 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmIonCompile.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/MathAlgorithms.h" + +#include + +#include "jit/ABIArgGenerator.h" +#include "jit/CodeGenerator.h" +#include "jit/CompileInfo.h" +#include "jit/Ion.h" +#include "jit/IonOptimizationLevels.h" +#include "jit/MIR.h" +#include "jit/ShuffleAnalysis.h" +#include "js/ScalarType.h" // js::Scalar::Type +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmGC.h" +#include "wasm/WasmGcObject.h" +#include "wasm/WasmGenerator.h" +#include "wasm/WasmIntrinsic.h" +#include "wasm/WasmOpIter.h" +#include "wasm/WasmSignalHandlers.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmValidate.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::IsPowerOfTwo; +using mozilla::Maybe; +using mozilla::Nothing; +using mozilla::Some; + +namespace { + +using BlockVector = Vector; +using DefVector = Vector; + +// To compile try-catch blocks, we extend the IonCompilePolicy's ControlItem +// from being just an MBasicBlock* to a Control structure collecting additional +// information. +using ControlInstructionVector = + Vector; + +struct Control { + MBasicBlock* block; + // For a try-catch ControlItem, when its block's Labelkind is Try, this + // collects branches to later bind and create the try's landing pad. + ControlInstructionVector tryPadPatches; + + Control() : block(nullptr) {} + + explicit Control(MBasicBlock* block) : block(block) {} + + public: + void setBlock(MBasicBlock* newBlock) { block = newBlock; } +}; + +// [SMDOC] WebAssembly Exception Handling in Ion +// ======================================================= +// +// ## Throwing instructions +// +// Wasm exceptions can be thrown by either a throw instruction (local throw), +// or by a wasm call. +// +// ## The "catching try control" +// +// We know we are in try-code if there is a surrounding ControlItem with +// LabelKind::Try. The innermost such control is called the +// "catching try control". +// +// ## Throws without a catching try control +// +// Such throws are implemented with an instance call that triggers the exception +// unwinding runtime. The exception unwinding runtime will not return to the +// function. +// +// ## "landing pad" and "pre-pad" blocks +// +// When an exception is thrown, the unwinder will search for the nearest +// enclosing try block and redirect control flow to it. The code that executes +// before any catch blocks is called the 'landing pad'. The 'landing pad' is +// responsible to: +// 1. Consume the pending exception state from +// Instance::pendingException(Tag) +// 2. Branch to the correct catch block, or else rethrow +// +// There is one landing pad for each try block. The immediate predecessors of +// the landing pad are called 'pre-pad' blocks. There is one pre-pad block per +// throwing instruction. +// +// ## Creating pre-pad blocks +// +// There are two possible sorts of pre-pad blocks, depending on whether we +// are branching after a local throw instruction, or after a wasm call: +// +// - If we encounter a local throw, we create the exception and tag objects, +// store them to Instance::pendingException(Tag), and then jump to the +// landing pad. +// +// - If we encounter a wasm call, we construct a MWasmCallCatchable which is a +// control instruction with either a branch to a fallthrough block or +// to a pre-pad block. +// +// The pre-pad block for a wasm call is empty except for a jump to the +// landing pad. It only exists to avoid critical edges which when split would +// violate the invariants of MWasmCallCatchable. The pending exception state +// is taken care of by the unwinder. +// +// Each pre-pad ends with a pending jump to the landing pad. The pending jumps +// to the landing pad are tracked in `tryPadPatches`. These are called +// "pad patches". +// +// ## Creating the landing pad +// +// When we exit try-code, we check if tryPadPatches has captured any control +// instructions (pad patches). If not, we don't compile any catches and we mark +// the rest as dead code. +// +// If there are pre-pad blocks, we join them to create a landing pad (or just +// "pad"). The pad's last two slots are the caught exception, and the +// exception's tag object. +// +// There are three different forms of try-catch/catch_all Wasm instructions, +// which result in different form of landing pad. +// +// 1. A catchless try, so a Wasm instruction of the form "try ... end". +// - In this case, we end the pad by rethrowing the caught exception. +// +// 2. A single catch_all after a try. +// - If the first catch after a try is a catch_all, then there won't be +// any more catches, but we need the exception and its tag object, in +// case the code in a catch_all contains "rethrow" instructions. +// - The Wasm instruction "rethrow", gets the exception and tag object to +// rethrow from the last two slots of the landing pad which, due to +// validation, is the l'th surrounding ControlItem. +// - We immediately GoTo to a new block after the pad and pop both the +// exception and tag object, as we don't need them anymore in this case. +// +// 3. Otherwise, there is one or more catch code blocks following. +// - In this case, we construct the landing pad by creating a sequence +// of compare and branch blocks that compare the pending exception tag +// object to the tag object of the current tagged catch block. This is +// done incrementally as we visit each tagged catch block in the bytecode +// stream. At every step, we update the ControlItem's block to point to +// the next block to be created in the landing pad sequence. The final +// block will either be a rethrow, if there is no catch_all, or else a +// jump to a catch_all block. + +struct IonCompilePolicy { + // We store SSA definitions in the value stack. + using Value = MDefinition*; + using ValueVector = DefVector; + + // We store loop headers and then/else blocks in the control flow stack. + // In the case of try-catch control blocks, we collect additional information + // regarding the possible paths from throws and calls to a landing pad, as + // well as information on the landing pad's handlers (its catches). + using ControlItem = Control; +}; + +using IonOpIter = OpIter; + +class FunctionCompiler; + +// CallCompileState describes a call that is being compiled. + +class CallCompileState { + // A generator object that is passed each argument as it is compiled. + WasmABIArgGenerator abi_; + + // Accumulates the register arguments while compiling arguments. + MWasmCallBase::Args regArgs_; + + // Reserved argument for passing Instance* to builtin instance method calls. + ABIArg instanceArg_; + + // The stack area in which the callee will write stack return values, or + // nullptr if no stack results. + MWasmStackResultArea* stackResultArea_ = nullptr; + + // Only FunctionCompiler should be directly manipulating CallCompileState. + friend class FunctionCompiler; +}; + +// Encapsulates the compilation of a single function in an asm.js module. The +// function compiler handles the creation and final backend compilation of the +// MIR graph. +class FunctionCompiler { + struct ControlFlowPatch { + MControlInstruction* ins; + uint32_t index; + ControlFlowPatch(MControlInstruction* ins, uint32_t index) + : ins(ins), index(index) {} + }; + + using ControlFlowPatchVector = Vector; + using ControlFlowPatchVectorVector = + Vector; + + const ModuleEnvironment& moduleEnv_; + IonOpIter iter_; + const FuncCompileInput& func_; + const ValTypeVector& locals_; + size_t lastReadCallSite_; + + TempAllocator& alloc_; + MIRGraph& graph_; + const CompileInfo& info_; + MIRGenerator& mirGen_; + + MBasicBlock* curBlock_; + uint32_t maxStackArgBytes_; + + uint32_t loopDepth_; + uint32_t blockDepth_; + ControlFlowPatchVectorVector blockPatches_; + + // Instance pointer argument to the current function. + MWasmParameter* instancePointer_; + MWasmParameter* stackResultPointer_; + + // Reference to masm.tryNotes_ + wasm::TryNoteVector& tryNotes_; + + public: + FunctionCompiler(const ModuleEnvironment& moduleEnv, Decoder& decoder, + const FuncCompileInput& func, const ValTypeVector& locals, + MIRGenerator& mirGen, TryNoteVector& tryNotes) + : moduleEnv_(moduleEnv), + iter_(moduleEnv, decoder), + func_(func), + locals_(locals), + lastReadCallSite_(0), + alloc_(mirGen.alloc()), + graph_(mirGen.graph()), + info_(mirGen.outerInfo()), + mirGen_(mirGen), + curBlock_(nullptr), + maxStackArgBytes_(0), + loopDepth_(0), + blockDepth_(0), + instancePointer_(nullptr), + stackResultPointer_(nullptr), + tryNotes_(tryNotes) {} + + const ModuleEnvironment& moduleEnv() const { return moduleEnv_; } + + IonOpIter& iter() { return iter_; } + TempAllocator& alloc() const { return alloc_; } + // FIXME(1401675): Replace with BlockType. + uint32_t funcIndex() const { return func_.index; } + const FuncType& funcType() const { + return *moduleEnv_.funcs[func_.index].type; + } + + BytecodeOffset bytecodeOffset() const { return iter_.bytecodeOffset(); } + BytecodeOffset bytecodeIfNotAsmJS() const { + return moduleEnv_.isAsmJS() ? BytecodeOffset() : iter_.bytecodeOffset(); + } + + [[nodiscard]] bool init() { + // Prepare the entry block for MIR generation: + + const ArgTypeVector args(funcType()); + + if (!mirGen_.ensureBallast()) { + return false; + } + if (!newBlock(/* prev */ nullptr, &curBlock_)) { + return false; + } + + for (WasmABIArgIter i(args); !i.done(); i++) { + MWasmParameter* ins = MWasmParameter::New(alloc(), *i, i.mirType()); + curBlock_->add(ins); + if (args.isSyntheticStackResultPointerArg(i.index())) { + MOZ_ASSERT(stackResultPointer_ == nullptr); + stackResultPointer_ = ins; + } else { + curBlock_->initSlot(info().localSlot(args.naturalIndex(i.index())), + ins); + } + if (!mirGen_.ensureBallast()) { + return false; + } + } + + // Set up a parameter that receives the hidden instance pointer argument. + instancePointer_ = + MWasmParameter::New(alloc(), ABIArg(InstanceReg), MIRType::Pointer); + curBlock_->add(instancePointer_); + if (!mirGen_.ensureBallast()) { + return false; + } + + for (size_t i = args.lengthWithoutStackResults(); i < locals_.length(); + i++) { + ValType slotValType = locals_[i]; +#ifndef ENABLE_WASM_SIMD + if (slotValType == ValType::V128) { + return iter().fail("Ion has no SIMD support yet"); + } +#endif + MDefinition* zero = constantZeroOfValType(slotValType); + curBlock_->initSlot(info().localSlot(i), zero); + if (!mirGen_.ensureBallast()) { + return false; + } + } + + return true; + } + + void finish() { + mirGen().initWasmMaxStackArgBytes(maxStackArgBytes_); + + MOZ_ASSERT(loopDepth_ == 0); + MOZ_ASSERT(blockDepth_ == 0); +#ifdef DEBUG + for (ControlFlowPatchVector& patches : blockPatches_) { + MOZ_ASSERT(patches.empty()); + } +#endif + MOZ_ASSERT(inDeadCode()); + MOZ_ASSERT(done(), "all bytes must be consumed"); + MOZ_ASSERT(func_.callSiteLineNums.length() == lastReadCallSite_); + } + + /************************* Read-only interface (after local scope setup) */ + + MIRGenerator& mirGen() const { return mirGen_; } + MIRGraph& mirGraph() const { return graph_; } + const CompileInfo& info() const { return info_; } + + MDefinition* getLocalDef(unsigned slot) { + if (inDeadCode()) { + return nullptr; + } + return curBlock_->getSlot(info().localSlot(slot)); + } + + const ValTypeVector& locals() const { return locals_; } + + /*********************************************************** Constants ***/ + + MDefinition* constantF32(float f) { + if (inDeadCode()) { + return nullptr; + } + auto* cst = MWasmFloatConstant::NewFloat32(alloc(), f); + curBlock_->add(cst); + return cst; + } + // Hide all other overloads, to guarantee no implicit argument conversion. + template + MDefinition* constantF32(T) = delete; + + MDefinition* constantF64(double d) { + if (inDeadCode()) { + return nullptr; + } + auto* cst = MWasmFloatConstant::NewDouble(alloc(), d); + curBlock_->add(cst); + return cst; + } + template + MDefinition* constantF64(T) = delete; + + MDefinition* constantI32(int32_t i) { + if (inDeadCode()) { + return nullptr; + } + MConstant* constant = + MConstant::New(alloc(), Int32Value(i), MIRType::Int32); + curBlock_->add(constant); + return constant; + } + template + MDefinition* constantI32(T) = delete; + + MDefinition* constantI64(int64_t i) { + if (inDeadCode()) { + return nullptr; + } + MConstant* constant = MConstant::NewInt64(alloc(), i); + curBlock_->add(constant); + return constant; + } + template + MDefinition* constantI64(T) = delete; + + // Produce an MConstant of the machine's target int type (Int32 or Int64). + MDefinition* constantTargetWord(intptr_t n) { + return targetIs64Bit() ? constantI64(int64_t(n)) : constantI32(int32_t(n)); + } + template + MDefinition* constantTargetWord(T) = delete; + +#ifdef ENABLE_WASM_SIMD + MDefinition* constantV128(V128 v) { + if (inDeadCode()) { + return nullptr; + } + MWasmFloatConstant* constant = MWasmFloatConstant::NewSimd128( + alloc(), SimdConstant::CreateSimd128((int8_t*)v.bytes)); + curBlock_->add(constant); + return constant; + } + template + MDefinition* constantV128(T) = delete; +#endif + + MDefinition* constantNullRef() { + if (inDeadCode()) { + return nullptr; + } + // MConstant has a lot of baggage so we don't use that here. + MWasmNullConstant* constant = MWasmNullConstant::New(alloc()); + curBlock_->add(constant); + return constant; + } + + // Produce a zero constant for the specified ValType. + MDefinition* constantZeroOfValType(ValType valType) { + switch (valType.kind()) { + case ValType::I32: + return constantI32(0); + case ValType::I64: + return constantI64(int64_t(0)); +#ifdef ENABLE_WASM_SIMD + case ValType::V128: + return constantV128(V128(0)); +#endif + case ValType::F32: + return constantF32(0.0f); + case ValType::F64: + return constantF64(0.0); + case ValType::Ref: + return constantNullRef(); + default: + MOZ_CRASH(); + } + } + + /***************************** Code generation (after local scope setup) */ + + void fence() { + if (inDeadCode()) { + return; + } + MWasmFence* ins = MWasmFence::New(alloc()); + curBlock_->add(ins); + } + + template + MDefinition* unary(MDefinition* op) { + if (inDeadCode()) { + return nullptr; + } + T* ins = T::New(alloc(), op); + curBlock_->add(ins); + return ins; + } + + template + MDefinition* unary(MDefinition* op, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + T* ins = T::New(alloc(), op, type); + curBlock_->add(ins); + return ins; + } + + template + MDefinition* binary(MDefinition* lhs, MDefinition* rhs) { + if (inDeadCode()) { + return nullptr; + } + T* ins = T::New(alloc(), lhs, rhs); + curBlock_->add(ins); + return ins; + } + + template + MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + T* ins = T::New(alloc(), lhs, rhs, type); + curBlock_->add(ins); + return ins; + } + + template + MDefinition* binary(MDefinition* lhs, MDefinition* rhs, MIRType type, + MWasmBinaryBitwise::SubOpcode subOpc) { + if (inDeadCode()) { + return nullptr; + } + T* ins = T::New(alloc(), lhs, rhs, type, subOpc); + curBlock_->add(ins); + return ins; + } + + MDefinition* ursh(MDefinition* lhs, MDefinition* rhs, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MUrsh::NewWasm(alloc(), lhs, rhs, type); + curBlock_->add(ins); + return ins; + } + + MDefinition* add(MDefinition* lhs, MDefinition* rhs, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MAdd::NewWasm(alloc(), lhs, rhs, type); + curBlock_->add(ins); + return ins; + } + + bool mustPreserveNaN(MIRType type) { + return IsFloatingPointType(type) && !moduleEnv().isAsmJS(); + } + + MDefinition* sub(MDefinition* lhs, MDefinition* rhs, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + + // wasm can't fold x - 0.0 because of NaN with custom payloads. + MSub* ins = MSub::NewWasm(alloc(), lhs, rhs, type, mustPreserveNaN(type)); + curBlock_->add(ins); + return ins; + } + + MDefinition* nearbyInt(MDefinition* input, RoundingMode roundingMode) { + if (inDeadCode()) { + return nullptr; + } + + auto* ins = MNearbyInt::New(alloc(), input, input->type(), roundingMode); + curBlock_->add(ins); + return ins; + } + + MDefinition* minMax(MDefinition* lhs, MDefinition* rhs, MIRType type, + bool isMax) { + if (inDeadCode()) { + return nullptr; + } + + if (mustPreserveNaN(type)) { + // Convert signaling NaN to quiet NaNs. + MDefinition* zero = constantZeroOfValType(ValType::fromMIRType(type)); + lhs = sub(lhs, zero, type); + rhs = sub(rhs, zero, type); + } + + MMinMax* ins = MMinMax::NewWasm(alloc(), lhs, rhs, type, isMax); + curBlock_->add(ins); + return ins; + } + + MDefinition* mul(MDefinition* lhs, MDefinition* rhs, MIRType type, + MMul::Mode mode) { + if (inDeadCode()) { + return nullptr; + } + + // wasm can't fold x * 1.0 because of NaN with custom payloads. + auto* ins = + MMul::NewWasm(alloc(), lhs, rhs, type, mode, mustPreserveNaN(type)); + curBlock_->add(ins); + return ins; + } + + MDefinition* div(MDefinition* lhs, MDefinition* rhs, MIRType type, + bool unsignd) { + if (inDeadCode()) { + return nullptr; + } + bool trapOnError = !moduleEnv().isAsmJS(); + if (!unsignd && type == MIRType::Int32) { + // Enforce the signedness of the operation by coercing the operands + // to signed. Otherwise, operands that "look" unsigned to Ion but + // are not unsigned to Baldr (eg, unsigned right shifts) may lead to + // the operation being executed unsigned. Applies to mod() as well. + // + // Do this for Int32 only since Int64 is not subject to the same + // issues. + // + // Note the offsets passed to MWasmBuiltinTruncateToInt32 are wrong here, + // but it doesn't matter: they're not codegen'd to calls since inputs + // already are int32. + auto* lhs2 = createTruncateToInt32(lhs); + curBlock_->add(lhs2); + lhs = lhs2; + auto* rhs2 = createTruncateToInt32(rhs); + curBlock_->add(rhs2); + rhs = rhs2; + } + + // For x86 and arm we implement i64 div via c++ builtin. + // A call to c++ builtin requires instance pointer. +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM) + if (type == MIRType::Int64) { + auto* ins = + MWasmBuiltinDivI64::New(alloc(), lhs, rhs, instancePointer_, unsignd, + trapOnError, bytecodeOffset()); + curBlock_->add(ins); + return ins; + } +#endif + + auto* ins = MDiv::New(alloc(), lhs, rhs, type, unsignd, trapOnError, + bytecodeOffset(), mustPreserveNaN(type)); + curBlock_->add(ins); + return ins; + } + + MInstruction* createTruncateToInt32(MDefinition* op) { + if (op->type() == MIRType::Double || op->type() == MIRType::Float32) { + return MWasmBuiltinTruncateToInt32::New(alloc(), op, instancePointer_); + } + + return MTruncateToInt32::New(alloc(), op); + } + + MDefinition* mod(MDefinition* lhs, MDefinition* rhs, MIRType type, + bool unsignd) { + if (inDeadCode()) { + return nullptr; + } + bool trapOnError = !moduleEnv().isAsmJS(); + if (!unsignd && type == MIRType::Int32) { + // See block comment in div(). + auto* lhs2 = createTruncateToInt32(lhs); + curBlock_->add(lhs2); + lhs = lhs2; + auto* rhs2 = createTruncateToInt32(rhs); + curBlock_->add(rhs2); + rhs = rhs2; + } + + // For x86 and arm we implement i64 mod via c++ builtin. + // A call to c++ builtin requires instance pointer. +#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM) + if (type == MIRType::Int64) { + auto* ins = + MWasmBuiltinModI64::New(alloc(), lhs, rhs, instancePointer_, unsignd, + trapOnError, bytecodeOffset()); + curBlock_->add(ins); + return ins; + } +#endif + + // Should be handled separately because we call BuiltinThunk for this case + // and so, need to add the dependency from instancePointer. + if (type == MIRType::Double) { + auto* ins = MWasmBuiltinModD::New(alloc(), lhs, rhs, instancePointer_, + type, bytecodeOffset()); + curBlock_->add(ins); + return ins; + } + + auto* ins = MMod::New(alloc(), lhs, rhs, type, unsignd, trapOnError, + bytecodeOffset()); + curBlock_->add(ins); + return ins; + } + + MDefinition* bitnot(MDefinition* op) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MBitNot::New(alloc(), op); + curBlock_->add(ins); + return ins; + } + + MDefinition* select(MDefinition* trueExpr, MDefinition* falseExpr, + MDefinition* condExpr) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MWasmSelect::New(alloc(), trueExpr, falseExpr, condExpr); + curBlock_->add(ins); + return ins; + } + + MDefinition* extendI32(MDefinition* op, bool isUnsigned) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MExtendInt32ToInt64::New(alloc(), op, isUnsigned); + curBlock_->add(ins); + return ins; + } + + MDefinition* signExtend(MDefinition* op, uint32_t srcSize, + uint32_t targetSize) { + if (inDeadCode()) { + return nullptr; + } + MInstruction* ins; + switch (targetSize) { + case 4: { + MSignExtendInt32::Mode mode; + switch (srcSize) { + case 1: + mode = MSignExtendInt32::Byte; + break; + case 2: + mode = MSignExtendInt32::Half; + break; + default: + MOZ_CRASH("Bad sign extension"); + } + ins = MSignExtendInt32::New(alloc(), op, mode); + break; + } + case 8: { + MSignExtendInt64::Mode mode; + switch (srcSize) { + case 1: + mode = MSignExtendInt64::Byte; + break; + case 2: + mode = MSignExtendInt64::Half; + break; + case 4: + mode = MSignExtendInt64::Word; + break; + default: + MOZ_CRASH("Bad sign extension"); + } + ins = MSignExtendInt64::New(alloc(), op, mode); + break; + } + default: { + MOZ_CRASH("Bad sign extension"); + } + } + curBlock_->add(ins); + return ins; + } + + MDefinition* convertI64ToFloatingPoint(MDefinition* op, MIRType type, + bool isUnsigned) { + if (inDeadCode()) { + return nullptr; + } +#if defined(JS_CODEGEN_ARM) + auto* ins = MBuiltinInt64ToFloatingPoint::New( + alloc(), op, instancePointer_, type, bytecodeOffset(), isUnsigned); +#else + auto* ins = MInt64ToFloatingPoint::New(alloc(), op, type, bytecodeOffset(), + isUnsigned); +#endif + curBlock_->add(ins); + return ins; + } + + MDefinition* rotate(MDefinition* input, MDefinition* count, MIRType type, + bool left) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MRotate::New(alloc(), input, count, type, left); + curBlock_->add(ins); + return ins; + } + + template + MDefinition* truncate(MDefinition* op, TruncFlags flags) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = T::New(alloc(), op, flags, bytecodeOffset()); + curBlock_->add(ins); + return ins; + } + +#if defined(JS_CODEGEN_ARM) + MDefinition* truncateWithInstance(MDefinition* op, TruncFlags flags) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MWasmBuiltinTruncateToInt64::New(alloc(), op, instancePointer_, + flags, bytecodeOffset()); + curBlock_->add(ins); + return ins; + } +#endif + + MDefinition* compare(MDefinition* lhs, MDefinition* rhs, JSOp op, + MCompare::CompareType type) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MCompare::NewWasm(alloc(), lhs, rhs, op, type); + curBlock_->add(ins); + return ins; + } + + void assign(unsigned slot, MDefinition* def) { + if (inDeadCode()) { + return; + } + curBlock_->setSlot(info().localSlot(slot), def); + } + + MDefinition* compareIsNull(MDefinition* value, JSOp compareOp) { + MDefinition* nullVal = constantNullRef(); + if (!nullVal) { + return nullptr; + } + return compare(value, nullVal, compareOp, MCompare::Compare_RefOrNull); + } + + [[nodiscard]] bool refAsNonNull(MDefinition* value) { + if (inDeadCode()) { + return true; + } + + auto* ins = MWasmTrapIfNull::New( + alloc(), value, wasm::Trap::NullPointerDereference, bytecodeOffset()); + + curBlock_->add(ins); + return true; + } + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + [[nodiscard]] bool brOnNull(uint32_t relativeDepth, const DefVector& values, + const ResultType& type, MDefinition* condition) { + if (inDeadCode()) { + return true; + } + + MBasicBlock* fallthroughBlock = nullptr; + if (!newBlock(curBlock_, &fallthroughBlock)) { + return false; + } + + MDefinition* check = compareIsNull(condition, JSOp::Eq); + if (!check) { + return false; + } + MTest* test = MTest::New(alloc(), check, nullptr, fallthroughBlock); + if (!test || + !addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) { + return false; + } + + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(test); + curBlock_ = fallthroughBlock; + return true; + } + + [[nodiscard]] bool brOnNonNull(uint32_t relativeDepth, + const DefVector& values, + const ResultType& type, + MDefinition* condition) { + if (inDeadCode()) { + return true; + } + + MBasicBlock* fallthroughBlock = nullptr; + if (!newBlock(curBlock_, &fallthroughBlock)) { + return false; + } + + MDefinition* check = compareIsNull(condition, JSOp::Ne); + if (!check) { + return false; + } + MTest* test = MTest::New(alloc(), check, nullptr, fallthroughBlock); + if (!test || + !addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) { + return false; + } + + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(test); + curBlock_ = fallthroughBlock; + return true; + } + +#endif // ENABLE_WASM_FUNCTION_REFERENCES + +#ifdef ENABLE_WASM_SIMD + // About Wasm SIMD as supported by Ion: + // + // The expectation is that Ion will only ever support SIMD on x86 and x64, + // since ARMv7 will cease to be a tier-1 platform soon, and MIPS64 will never + // implement SIMD. + // + // The division of the operations into MIR nodes reflects that expectation, + // and is a good fit for x86/x64. Should the expectation change we'll + // possibly want to re-architect the SIMD support to be a little more general. + // + // Most SIMD operations map directly to a single MIR node that ultimately ends + // up being expanded in the macroassembler. + // + // Some SIMD operations that do have a complete macroassembler expansion are + // open-coded into multiple MIR nodes here; in some cases that's just + // convenience, in other cases it may also allow them to benefit from Ion + // optimizations. The reason for the expansions will be documented by a + // comment. + + // (v128,v128) -> v128 effect-free binary operations + MDefinition* binarySimd128(MDefinition* lhs, MDefinition* rhs, + bool commutative, SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(lhs->type() == MIRType::Simd128 && + rhs->type() == MIRType::Simd128); + + auto* ins = MWasmBinarySimd128::New(alloc(), lhs, rhs, commutative, op); + curBlock_->add(ins); + return ins; + } + + // (v128,i32) -> v128 effect-free shift operations + MDefinition* shiftSimd128(MDefinition* lhs, MDefinition* rhs, SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(lhs->type() == MIRType::Simd128 && + rhs->type() == MIRType::Int32); + + int32_t maskBits; + if (MacroAssembler::MustMaskShiftCountSimd128(op, &maskBits)) { + MDefinition* mask = constantI32(maskBits); + auto* rhs2 = MBitAnd::New(alloc(), rhs, mask, MIRType::Int32); + curBlock_->add(rhs2); + rhs = rhs2; + } + + auto* ins = MWasmShiftSimd128::New(alloc(), lhs, rhs, op); + curBlock_->add(ins); + return ins; + } + + // (v128,scalar,imm) -> v128 + MDefinition* replaceLaneSimd128(MDefinition* lhs, MDefinition* rhs, + uint32_t laneIndex, SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(lhs->type() == MIRType::Simd128); + + auto* ins = MWasmReplaceLaneSimd128::New(alloc(), lhs, rhs, laneIndex, op); + curBlock_->add(ins); + return ins; + } + + // (scalar) -> v128 effect-free unary operations + MDefinition* scalarToSimd128(MDefinition* src, SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + auto* ins = MWasmScalarToSimd128::New(alloc(), src, op); + curBlock_->add(ins); + return ins; + } + + // (v128) -> v128 effect-free unary operations + MDefinition* unarySimd128(MDefinition* src, SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(src->type() == MIRType::Simd128); + auto* ins = MWasmUnarySimd128::New(alloc(), src, op); + curBlock_->add(ins); + return ins; + } + + // (v128, imm) -> scalar effect-free unary operations + MDefinition* reduceSimd128(MDefinition* src, SimdOp op, ValType outType, + uint32_t imm = 0) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(src->type() == MIRType::Simd128); + auto* ins = + MWasmReduceSimd128::New(alloc(), src, op, outType.toMIRType(), imm); + curBlock_->add(ins); + return ins; + } + + // (v128, v128, v128) -> v128 effect-free operations + MDefinition* ternarySimd128(MDefinition* v0, MDefinition* v1, MDefinition* v2, + SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(v0->type() == MIRType::Simd128 && + v1->type() == MIRType::Simd128 && + v2->type() == MIRType::Simd128); + + auto* ins = MWasmTernarySimd128::New(alloc(), v0, v1, v2, op); + curBlock_->add(ins); + return ins; + } + + // (v128, v128, imm_v128) -> v128 effect-free operations + MDefinition* shuffleSimd128(MDefinition* v1, MDefinition* v2, V128 control) { + if (inDeadCode()) { + return nullptr; + } + + MOZ_ASSERT(v1->type() == MIRType::Simd128); + MOZ_ASSERT(v2->type() == MIRType::Simd128); + auto* ins = BuildWasmShuffleSimd128( + alloc(), reinterpret_cast(control.bytes), v1, v2); + curBlock_->add(ins); + return ins; + } + + // Also see below for SIMD memory references + +#endif // ENABLE_WASM_SIMD + + /************************************************ Linear memory accesses */ + + // For detailed information about memory accesses, see "Linear memory + // addresses and bounds checking" in WasmMemory.cpp. + + private: + // If the platform does not have a HeapReg, load the memory base from + // instance. + MWasmLoadInstance* maybeLoadMemoryBase() { + MWasmLoadInstance* load = nullptr; +#ifdef JS_CODEGEN_X86 + AliasSet aliases = !moduleEnv_.memory->canMovingGrow() + ? AliasSet::None() + : AliasSet::Load(AliasSet::WasmHeapMeta); + load = MWasmLoadInstance::New(alloc(), instancePointer_, + wasm::Instance::offsetOfMemoryBase(), + MIRType::Pointer, aliases); + curBlock_->add(load); +#endif + return load; + } + + public: + // A value holding the memory base, whether that's HeapReg or some other + // register. + MWasmHeapBase* memoryBase() { + MWasmHeapBase* base = nullptr; + AliasSet aliases = !moduleEnv_.memory->canMovingGrow() + ? AliasSet::None() + : AliasSet::Load(AliasSet::WasmHeapMeta); + base = MWasmHeapBase::New(alloc(), instancePointer_, aliases); + curBlock_->add(base); + return base; + } + + private: + // If the bounds checking strategy requires it, load the bounds check limit + // from the instance. + MWasmLoadInstance* maybeLoadBoundsCheckLimit(MIRType type) { + MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Int64); + if (moduleEnv_.hugeMemoryEnabled()) { + return nullptr; + } + AliasSet aliases = !moduleEnv_.memory->canMovingGrow() + ? AliasSet::None() + : AliasSet::Load(AliasSet::WasmHeapMeta); + auto* load = MWasmLoadInstance::New( + alloc(), instancePointer_, wasm::Instance::offsetOfBoundsCheckLimit(), + type, aliases); + curBlock_->add(load); + return load; + } + + // Return true if the access requires an alignment check. If so, sets + // *mustAdd to true if the offset must be added to the pointer before + // checking. + bool needAlignmentCheck(MemoryAccessDesc* access, MDefinition* base, + bool* mustAdd) { + MOZ_ASSERT(!*mustAdd); + + // asm.js accesses are always aligned and need no checks. + if (moduleEnv_.isAsmJS() || !access->isAtomic()) { + return false; + } + + // If the EA is known and aligned it will need no checks. + if (base->isConstant()) { + // We only care about the low bits, so overflow is OK, as is chopping off + // the high bits of an i64 pointer. + uint32_t ptr = 0; + if (isMem64()) { + ptr = uint32_t(base->toConstant()->toInt64()); + } else { + ptr = base->toConstant()->toInt32(); + } + if (((ptr + access->offset64()) & (access->byteSize() - 1)) == 0) { + return false; + } + } + + // If the offset is aligned then the EA is just the pointer, for + // the purposes of this check. + *mustAdd = (access->offset64() & (access->byteSize() - 1)) != 0; + return true; + } + + // Fold a constant base into the offset and make the base 0, provided the + // offset stays below the guard limit. The reason for folding the base into + // the offset rather than vice versa is that a small offset can be ignored + // by both explicit bounds checking and bounds check elimination. + void foldConstantPointer(MemoryAccessDesc* access, MDefinition** base) { + uint32_t offsetGuardLimit = + GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled()); + + if ((*base)->isConstant()) { + uint64_t basePtr = 0; + if (isMem64()) { + basePtr = uint64_t((*base)->toConstant()->toInt64()); + } else { + basePtr = uint64_t(int64_t((*base)->toConstant()->toInt32())); + } + + uint64_t offset = access->offset64(); + + if (offset < offsetGuardLimit && basePtr < offsetGuardLimit - offset) { + offset += uint32_t(basePtr); + access->setOffset32(uint32_t(offset)); + *base = isMem64() ? constantI64(int64_t(0)) : constantI32(0); + } + } + } + + // If the offset must be added because it is large or because the true EA must + // be checked, compute the effective address, trapping on overflow. + void maybeComputeEffectiveAddress(MemoryAccessDesc* access, + MDefinition** base, bool mustAddOffset) { + uint32_t offsetGuardLimit = + GetMaxOffsetGuardLimit(moduleEnv_.hugeMemoryEnabled()); + + if (access->offset64() >= offsetGuardLimit || + access->offset64() > UINT32_MAX || mustAddOffset || + !JitOptions.wasmFoldOffsets) { + *base = computeEffectiveAddress(*base, access); + } + } + + MWasmLoadInstance* needBoundsCheck() { +#ifdef JS_64BIT + // For 32-bit base pointers: + // + // If the bounds check uses the full 64 bits of the bounds check limit, then + // the base pointer must be zero-extended to 64 bits before checking and + // wrapped back to 32-bits after Spectre masking. (And it's important that + // the value we end up with has flowed through the Spectre mask.) + // + // If the memory's max size is known to be smaller than 64K pages exactly, + // we can use a 32-bit check and avoid extension and wrapping. + static_assert(0x100000000 % PageSize == 0); + bool mem32LimitIs64Bits = isMem32() && + !moduleEnv_.memory->boundsCheckLimitIs32Bits() && + MaxMemoryPages(moduleEnv_.memory->indexType()) >= + Pages(0x100000000 / PageSize); +#else + // On 32-bit platforms we have no more than 2GB memory and the limit for a + // 32-bit base pointer is never a 64-bit value. + bool mem32LimitIs64Bits = false; +#endif + return maybeLoadBoundsCheckLimit( + mem32LimitIs64Bits || isMem64() ? MIRType::Int64 : MIRType::Int32); + } + + void performBoundsCheck(MDefinition** base, + MWasmLoadInstance* boundsCheckLimit) { + // At the outset, actualBase could be the result of pretty much any integer + // operation, or it could be the load of an integer constant. If its type + // is i32, we may assume the value has a canonical representation for the + // platform, see doc block in MacroAssembler.h. + MDefinition* actualBase = *base; + + // Extend an i32 index value to perform a 64-bit bounds check if the memory + // can be 4GB or larger. + bool extendAndWrapIndex = + isMem32() && boundsCheckLimit->type() == MIRType::Int64; + if (extendAndWrapIndex) { + auto* extended = MWasmExtendU32Index::New(alloc(), actualBase); + curBlock_->add(extended); + actualBase = extended; + } + + auto* ins = + MWasmBoundsCheck::New(alloc(), actualBase, boundsCheckLimit, + bytecodeOffset(), MWasmBoundsCheck::Memory0); + curBlock_->add(ins); + actualBase = ins; + + // If we're masking, then we update *base to create a dependency chain + // through the masked index. But we will first need to wrap the index + // value if it was extended above. + if (JitOptions.spectreIndexMasking) { + if (extendAndWrapIndex) { + auto* wrapped = MWasmWrapU32Index::New(alloc(), actualBase); + curBlock_->add(wrapped); + actualBase = wrapped; + } + *base = actualBase; + } + } + + // Perform all necessary checking before a wasm heap access, based on the + // attributes of the access and base pointer. + // + // For 64-bit indices on platforms that are limited to indices that fit into + // 32 bits (all 32-bit platforms and mips64), this returns a bounds-checked + // `base` that has type Int32. Lowering code depends on this and will assert + // that the base has this type. See the end of this function. + + void checkOffsetAndAlignmentAndBounds(MemoryAccessDesc* access, + MDefinition** base) { + MOZ_ASSERT(!inDeadCode()); + MOZ_ASSERT(!moduleEnv_.isAsmJS()); + + // Attempt to fold an offset into a constant base pointer so as to simplify + // the addressing expression. This may update *base. + foldConstantPointer(access, base); + + // Determine whether an alignment check is needed and whether the offset + // must be checked too. + bool mustAddOffsetForAlignmentCheck = false; + bool alignmentCheck = + needAlignmentCheck(access, *base, &mustAddOffsetForAlignmentCheck); + + // If bounds checking or alignment checking requires it, compute the + // effective address: add the offset into the pointer and trap on overflow. + // This may update *base. + maybeComputeEffectiveAddress(access, base, mustAddOffsetForAlignmentCheck); + + // Emit the alignment check if necessary; it traps if it fails. + if (alignmentCheck) { + curBlock_->add(MWasmAlignmentCheck::New( + alloc(), *base, access->byteSize(), bytecodeOffset())); + } + + // Emit the bounds check if necessary; it traps if it fails. This may + // update *base. + MWasmLoadInstance* boundsCheckLimit = needBoundsCheck(); + if (boundsCheckLimit) { + performBoundsCheck(base, boundsCheckLimit); + } + +#ifndef JS_64BIT + if (isMem64()) { + // We must have had an explicit bounds check (or one was elided if it was + // proved redundant), and on 32-bit systems the index will for sure fit in + // 32 bits: the max memory is 2GB. So chop the index down to 32-bit to + // simplify the back-end. + MOZ_ASSERT((*base)->type() == MIRType::Int64); + MOZ_ASSERT(!moduleEnv_.hugeMemoryEnabled()); + auto* chopped = MWasmWrapU32Index::New(alloc(), *base); + MOZ_ASSERT(chopped->type() == MIRType::Int32); + curBlock_->add(chopped); + *base = chopped; + } +#endif + } + + bool isSmallerAccessForI64(ValType result, const MemoryAccessDesc* access) { + if (result == ValType::I64 && access->byteSize() <= 4) { + // These smaller accesses should all be zero-extending. + MOZ_ASSERT(!isSignedIntType(access->type())); + return true; + } + return false; + } + + public: + bool isMem32() { return moduleEnv_.memory->indexType() == IndexType::I32; } + bool isMem64() { return moduleEnv_.memory->indexType() == IndexType::I64; } + + // Sometimes, we need to determine the memory type before the opcode reader + // that will reject a memory opcode in the presence of no-memory gets a chance + // to do so. This predicate is safe. + bool isNoMemOrMem32() { + return !moduleEnv_.usesMemory() || + moduleEnv_.memory->indexType() == IndexType::I32; + } + + // Add the offset into the pointer to yield the EA; trap on overflow. + MDefinition* computeEffectiveAddress(MDefinition* base, + MemoryAccessDesc* access) { + if (inDeadCode()) { + return nullptr; + } + uint64_t offset = access->offset64(); + if (offset == 0) { + return base; + } + auto* ins = MWasmAddOffset::New(alloc(), base, offset, bytecodeOffset()); + curBlock_->add(ins); + access->clearOffset(); + return ins; + } + + MDefinition* load(MDefinition* base, MemoryAccessDesc* access, + ValType result) { + if (inDeadCode()) { + return nullptr; + } + + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MInstruction* load = nullptr; + if (moduleEnv_.isAsmJS()) { + MOZ_ASSERT(access->offset64() == 0); + MWasmLoadInstance* boundsCheckLimit = + maybeLoadBoundsCheckLimit(MIRType::Int32); + load = MAsmJSLoadHeap::New(alloc(), memoryBase, base, boundsCheckLimit, + access->type()); + } else { + checkOffsetAndAlignmentAndBounds(access, &base); +#ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +#endif + load = MWasmLoad::New(alloc(), memoryBase, base, *access, + result.toMIRType()); + } + if (!load) { + return nullptr; + } + curBlock_->add(load); + return load; + } + + void store(MDefinition* base, MemoryAccessDesc* access, MDefinition* v) { + if (inDeadCode()) { + return; + } + + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MInstruction* store = nullptr; + if (moduleEnv_.isAsmJS()) { + MOZ_ASSERT(access->offset64() == 0); + MWasmLoadInstance* boundsCheckLimit = + maybeLoadBoundsCheckLimit(MIRType::Int32); + store = MAsmJSStoreHeap::New(alloc(), memoryBase, base, boundsCheckLimit, + access->type(), v); + } else { + checkOffsetAndAlignmentAndBounds(access, &base); +#ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +#endif + store = MWasmStore::New(alloc(), memoryBase, base, *access, v); + } + if (!store) { + return; + } + curBlock_->add(store); + } + + MDefinition* atomicCompareExchangeHeap(MDefinition* base, + MemoryAccessDesc* access, + ValType result, MDefinition* oldv, + MDefinition* newv) { + if (inDeadCode()) { + return nullptr; + } + + checkOffsetAndAlignmentAndBounds(access, &base); +#ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +#endif + + if (isSmallerAccessForI64(result, access)) { + auto* cvtOldv = + MWrapInt64ToInt32::New(alloc(), oldv, /*bottomHalf=*/true); + curBlock_->add(cvtOldv); + oldv = cvtOldv; + + auto* cvtNewv = + MWrapInt64ToInt32::New(alloc(), newv, /*bottomHalf=*/true); + curBlock_->add(cvtNewv); + newv = cvtNewv; + } + + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MInstruction* cas = MWasmCompareExchangeHeap::New( + alloc(), bytecodeOffset(), memoryBase, base, *access, oldv, newv, + instancePointer_); + if (!cas) { + return nullptr; + } + curBlock_->add(cas); + + if (isSmallerAccessForI64(result, access)) { + cas = MExtendInt32ToInt64::New(alloc(), cas, true); + curBlock_->add(cas); + } + + return cas; + } + + MDefinition* atomicExchangeHeap(MDefinition* base, MemoryAccessDesc* access, + ValType result, MDefinition* value) { + if (inDeadCode()) { + return nullptr; + } + + checkOffsetAndAlignmentAndBounds(access, &base); +#ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +#endif + + if (isSmallerAccessForI64(result, access)) { + auto* cvtValue = + MWrapInt64ToInt32::New(alloc(), value, /*bottomHalf=*/true); + curBlock_->add(cvtValue); + value = cvtValue; + } + + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MInstruction* xchg = + MWasmAtomicExchangeHeap::New(alloc(), bytecodeOffset(), memoryBase, + base, *access, value, instancePointer_); + if (!xchg) { + return nullptr; + } + curBlock_->add(xchg); + + if (isSmallerAccessForI64(result, access)) { + xchg = MExtendInt32ToInt64::New(alloc(), xchg, true); + curBlock_->add(xchg); + } + + return xchg; + } + + MDefinition* atomicBinopHeap(AtomicOp op, MDefinition* base, + MemoryAccessDesc* access, ValType result, + MDefinition* value) { + if (inDeadCode()) { + return nullptr; + } + + checkOffsetAndAlignmentAndBounds(access, &base); +#ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +#endif + + if (isSmallerAccessForI64(result, access)) { + auto* cvtValue = + MWrapInt64ToInt32::New(alloc(), value, /*bottomHalf=*/true); + curBlock_->add(cvtValue); + value = cvtValue; + } + + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MInstruction* binop = + MWasmAtomicBinopHeap::New(alloc(), bytecodeOffset(), op, memoryBase, + base, *access, value, instancePointer_); + if (!binop) { + return nullptr; + } + curBlock_->add(binop); + + if (isSmallerAccessForI64(result, access)) { + binop = MExtendInt32ToInt64::New(alloc(), binop, true); + curBlock_->add(binop); + } + + return binop; + } + +#ifdef ENABLE_WASM_SIMD + MDefinition* loadSplatSimd128(Scalar::Type viewType, + const LinearMemoryAddress& addr, + wasm::SimdOp splatOp) { + if (inDeadCode()) { + return nullptr; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, + bytecodeIfNotAsmJS()); + + // Generate better code (on x86) + // If AVX2 is enabled, more broadcast operators are available. + if (viewType == Scalar::Float64 +# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) + || (js::jit::CPUInfo::IsAVX2Present() && + (viewType == Scalar::Uint8 || viewType == Scalar::Uint16 || + viewType == Scalar::Float32)) +# endif + ) { + access.setSplatSimd128Load(); + return load(addr.base, &access, ValType::V128); + } + + ValType resultType = ValType::I32; + if (viewType == Scalar::Float32) { + resultType = ValType::F32; + splatOp = wasm::SimdOp::F32x4Splat; + } + auto* scalar = load(addr.base, &access, resultType); + if (!inDeadCode() && !scalar) { + return nullptr; + } + return scalarToSimd128(scalar, splatOp); + } + + MDefinition* loadExtendSimd128(const LinearMemoryAddress& addr, + wasm::SimdOp op) { + if (inDeadCode()) { + return nullptr; + } + + // Generate better code (on x86) by loading as a double with an + // operation that sign extends directly. + MemoryAccessDesc access(Scalar::Float64, addr.align, addr.offset, + bytecodeIfNotAsmJS()); + access.setWidenSimd128Load(op); + return load(addr.base, &access, ValType::V128); + } + + MDefinition* loadZeroSimd128(Scalar::Type viewType, size_t numBytes, + const LinearMemoryAddress& addr) { + if (inDeadCode()) { + return nullptr; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, + bytecodeIfNotAsmJS()); + access.setZeroExtendSimd128Load(); + return load(addr.base, &access, ValType::V128); + } + + MDefinition* loadLaneSimd128(uint32_t laneSize, + const LinearMemoryAddress& addr, + uint32_t laneIndex, MDefinition* src) { + if (inDeadCode()) { + return nullptr; + } + + MemoryAccessDesc access(Scalar::Simd128, addr.align, addr.offset, + bytecodeIfNotAsmJS()); + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MDefinition* base = addr.base; + MOZ_ASSERT(!moduleEnv_.isAsmJS()); + checkOffsetAndAlignmentAndBounds(&access, &base); +# ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +# endif + MInstruction* load = MWasmLoadLaneSimd128::New( + alloc(), memoryBase, base, access, laneSize, laneIndex, src); + if (!load) { + return nullptr; + } + curBlock_->add(load); + return load; + } + + void storeLaneSimd128(uint32_t laneSize, + const LinearMemoryAddress& addr, + uint32_t laneIndex, MDefinition* src) { + if (inDeadCode()) { + return; + } + MemoryAccessDesc access(Scalar::Simd128, addr.align, addr.offset, + bytecodeIfNotAsmJS()); + MWasmLoadInstance* memoryBase = maybeLoadMemoryBase(); + MDefinition* base = addr.base; + MOZ_ASSERT(!moduleEnv_.isAsmJS()); + checkOffsetAndAlignmentAndBounds(&access, &base); +# ifndef JS_64BIT + MOZ_ASSERT(base->type() == MIRType::Int32); +# endif + MInstruction* store = MWasmStoreLaneSimd128::New( + alloc(), memoryBase, base, access, laneSize, laneIndex, src); + if (!store) { + return; + } + curBlock_->add(store); + } +#endif // ENABLE_WASM_SIMD + + /************************************************ Global variable accesses */ + + MDefinition* loadGlobalVar(unsigned instanceDataOffset, bool isConst, + bool isIndirect, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + + MInstruction* load; + if (isIndirect) { + // Pull a pointer to the value out of Instance::globalArea, then + // load from that pointer. Note that the pointer is immutable + // even though the value it points at may change, hence the use of + // |true| for the first node's |isConst| value, irrespective of + // the |isConst| formal parameter to this method. The latter + // applies to the denoted value as a whole. + auto* cellPtr = MWasmLoadInstanceDataField::New( + alloc(), MIRType::Pointer, instanceDataOffset, + /*isConst=*/true, instancePointer_); + curBlock_->add(cellPtr); + load = MWasmLoadGlobalCell::New(alloc(), type, cellPtr); + } else { + // Pull the value directly out of Instance::globalArea. + load = MWasmLoadInstanceDataField::New(alloc(), type, instanceDataOffset, + isConst, instancePointer_); + } + curBlock_->add(load); + return load; + } + + [[nodiscard]] bool storeGlobalVar(uint32_t lineOrBytecode, + uint32_t instanceDataOffset, + bool isIndirect, MDefinition* v) { + if (inDeadCode()) { + return true; + } + + if (isIndirect) { + // Pull a pointer to the value out of Instance::globalArea, then + // store through that pointer. + auto* valueAddr = MWasmLoadInstanceDataField::New( + alloc(), MIRType::Pointer, instanceDataOffset, + /*isConst=*/true, instancePointer_); + curBlock_->add(valueAddr); + + // Handle a store to a ref-typed field specially + if (v->type() == MIRType::RefOrNull) { + // Load the previous value for the post-write barrier + auto* prevValue = + MWasmLoadGlobalCell::New(alloc(), MIRType::RefOrNull, valueAddr); + curBlock_->add(prevValue); + + // Store the new value + auto* store = + MWasmStoreRef::New(alloc(), instancePointer_, valueAddr, + /*valueOffset=*/0, v, AliasSet::WasmGlobalCell, + WasmPreBarrierKind::Normal); + curBlock_->add(store); + + // Call the post-write barrier + return postBarrierPrecise(lineOrBytecode, valueAddr, prevValue); + } + + auto* store = MWasmStoreGlobalCell::New(alloc(), v, valueAddr); + curBlock_->add(store); + return true; + } + // Or else store the value directly in Instance::globalArea. + + // Handle a store to a ref-typed field specially + if (v->type() == MIRType::RefOrNull) { + // Compute the address of the ref-typed global + auto* valueAddr = MWasmDerivedPointer::New( + alloc(), instancePointer_, + wasm::Instance::offsetInData(instanceDataOffset)); + curBlock_->add(valueAddr); + + // Load the previous value for the post-write barrier + auto* prevValue = + MWasmLoadGlobalCell::New(alloc(), MIRType::RefOrNull, valueAddr); + curBlock_->add(prevValue); + + // Store the new value + auto* store = + MWasmStoreRef::New(alloc(), instancePointer_, valueAddr, + /*valueOffset=*/0, v, AliasSet::WasmInstanceData, + WasmPreBarrierKind::Normal); + curBlock_->add(store); + + // Call the post-write barrier + return postBarrierPrecise(lineOrBytecode, valueAddr, prevValue); + } + + auto* store = MWasmStoreInstanceDataField::New(alloc(), instanceDataOffset, + v, instancePointer_); + curBlock_->add(store); + return true; + } + + MDefinition* loadTableField(uint32_t tableIndex, unsigned fieldOffset, + MIRType type) { + uint32_t instanceDataOffset = wasm::Instance::offsetInData( + moduleEnv_.offsetOfTableInstanceData(tableIndex) + fieldOffset); + auto* load = + MWasmLoadInstance::New(alloc(), instancePointer_, instanceDataOffset, + type, AliasSet::Load(AliasSet::WasmTableMeta)); + curBlock_->add(load); + return load; + } + + MDefinition* loadTableLength(uint32_t tableIndex) { + return loadTableField(tableIndex, offsetof(TableInstanceData, length), + MIRType::Int32); + } + + MDefinition* loadTableElements(uint32_t tableIndex) { + return loadTableField(tableIndex, offsetof(TableInstanceData, elements), + MIRType::Pointer); + } + + MDefinition* tableGetAnyRef(uint32_t tableIndex, MDefinition* index) { + // Load the table length and perform a bounds check with spectre index + // masking + auto* length = loadTableLength(tableIndex); + auto* check = MWasmBoundsCheck::New( + alloc(), index, length, bytecodeOffset(), MWasmBoundsCheck::Unknown); + curBlock_->add(check); + if (JitOptions.spectreIndexMasking) { + index = check; + } + + // Load the table elements and load the element + auto* elements = loadTableElements(tableIndex); + auto* element = MWasmLoadTableElement::New(alloc(), elements, index); + curBlock_->add(element); + return element; + } + + [[nodiscard]] bool tableSetAnyRef(uint32_t tableIndex, MDefinition* index, + MDefinition* value, + uint32_t lineOrBytecode) { + // Load the table length and perform a bounds check with spectre index + // masking + auto* length = loadTableLength(tableIndex); + auto* check = MWasmBoundsCheck::New( + alloc(), index, length, bytecodeOffset(), MWasmBoundsCheck::Unknown); + curBlock_->add(check); + if (JitOptions.spectreIndexMasking) { + index = check; + } + + // Load the table elements + auto* elements = loadTableElements(tableIndex); + + // Load the previous value + auto* prevValue = MWasmLoadTableElement::New(alloc(), elements, index); + curBlock_->add(prevValue); + + // Compute the value's location for the post barrier + auto* loc = + MWasmDerivedIndexPointer::New(alloc(), elements, index, ScalePointer); + curBlock_->add(loc); + + // Store the new value + auto* store = MWasmStoreRef::New( + alloc(), instancePointer_, loc, /*valueOffset=*/0, value, + AliasSet::WasmTableElement, WasmPreBarrierKind::Normal); + curBlock_->add(store); + + // Perform the post barrier + return postBarrierPrecise(lineOrBytecode, loc, prevValue); + } + + void addInterruptCheck() { + if (inDeadCode()) { + return; + } + curBlock_->add( + MWasmInterruptCheck::New(alloc(), instancePointer_, bytecodeOffset())); + } + + // Perform a post-write barrier to update the generational store buffer. This + // version will remove a previous store buffer entry if it is no longer + // needed. + [[nodiscard]] bool postBarrierPrecise(uint32_t lineOrBytecode, + MDefinition* valueAddr, + MDefinition* value) { + return emitInstanceCall2(lineOrBytecode, SASigPostBarrierPrecise, valueAddr, + value); + } + + // Perform a post-write barrier to update the generational store buffer. This + // version will remove a previous store buffer entry if it is no longer + // needed. + [[nodiscard]] bool postBarrierPreciseWithOffset(uint32_t lineOrBytecode, + MDefinition* valueBase, + uint32_t valueOffset, + MDefinition* value) { + MDefinition* valueOffsetDef = constantI32(int32_t(valueOffset)); + if (!valueOffsetDef) { + return false; + } + return emitInstanceCall3(lineOrBytecode, SASigPostBarrierPreciseWithOffset, + valueBase, valueOffsetDef, value); + } + + // Perform a post-write barrier to update the generational store buffer. This + // version is the most efficient and only requires the address to store the + // value and the new value. It does not remove a previous store buffer entry + // if it is no longer needed, you must use a precise post-write barrier for + // that. + [[nodiscard]] bool postBarrier(uint32_t lineOrBytecode, MDefinition* object, + MDefinition* valueBase, uint32_t valueOffset, + MDefinition* newValue) { + auto* barrier = MWasmPostWriteBarrier::New( + alloc(), instancePointer_, object, valueBase, valueOffset, newValue); + if (!barrier) { + return false; + } + curBlock_->add(barrier); + return true; + } + + /***************************************************************** Calls */ + + // The IonMonkey backend maintains a single stack offset (from the stack + // pointer to the base of the frame) by adding the total amount of spill + // space required plus the maximum stack required for argument passing. + // Since we do not use IonMonkey's MPrepareCall/MPassArg/MCall, we must + // manually accumulate, for the entire function, the maximum required stack + // space for argument passing. (This is passed to the CodeGenerator via + // MIRGenerator::maxWasmStackArgBytes.) This is just be the maximum of the + // stack space required for each individual call (as determined by the call + // ABI). + + // Operations that modify a CallCompileState. + + [[nodiscard]] bool passInstance(MIRType instanceType, + CallCompileState* args) { + if (inDeadCode()) { + return true; + } + + // Should only pass an instance once. And it must be a non-GC pointer. + MOZ_ASSERT(args->instanceArg_ == ABIArg()); + MOZ_ASSERT(instanceType == MIRType::Pointer); + args->instanceArg_ = args->abi_.next(MIRType::Pointer); + return true; + } + + // Do not call this directly. Call one of the passArg() variants instead. + [[nodiscard]] bool passArgWorker(MDefinition* argDef, MIRType type, + CallCompileState* call) { + ABIArg arg = call->abi_.next(type); + switch (arg.kind()) { +#ifdef JS_CODEGEN_REGISTER_PAIR + case ABIArg::GPR_PAIR: { + auto mirLow = + MWrapInt64ToInt32::New(alloc(), argDef, /* bottomHalf = */ true); + curBlock_->add(mirLow); + auto mirHigh = + MWrapInt64ToInt32::New(alloc(), argDef, /* bottomHalf = */ false); + curBlock_->add(mirHigh); + return call->regArgs_.append( + MWasmCallBase::Arg(AnyRegister(arg.gpr64().low), mirLow)) && + call->regArgs_.append( + MWasmCallBase::Arg(AnyRegister(arg.gpr64().high), mirHigh)); + } +#endif + case ABIArg::GPR: + case ABIArg::FPU: + return call->regArgs_.append(MWasmCallBase::Arg(arg.reg(), argDef)); + case ABIArg::Stack: { + auto* mir = + MWasmStackArg::New(alloc(), arg.offsetFromArgBase(), argDef); + curBlock_->add(mir); + return true; + } + case ABIArg::Uninitialized: + MOZ_ASSERT_UNREACHABLE("Uninitialized ABIArg kind"); + } + MOZ_CRASH("Unknown ABIArg kind."); + } + + template + [[nodiscard]] bool passArgs(const DefVector& argDefs, SpanT types, + CallCompileState* call) { + MOZ_ASSERT(argDefs.length() == types.size()); + for (uint32_t i = 0; i < argDefs.length(); i++) { + MDefinition* def = argDefs[i]; + ValType type = types[i]; + if (!passArg(def, type, call)) { + return false; + } + } + return true; + } + + [[nodiscard]] bool passArg(MDefinition* argDef, MIRType type, + CallCompileState* call) { + if (inDeadCode()) { + return true; + } + return passArgWorker(argDef, type, call); + } + + [[nodiscard]] bool passArg(MDefinition* argDef, ValType type, + CallCompileState* call) { + if (inDeadCode()) { + return true; + } + return passArgWorker(argDef, type.toMIRType(), call); + } + + // If the call returns results on the stack, prepare a stack area to receive + // them, and pass the address of the stack area to the callee as an additional + // argument. + [[nodiscard]] bool passStackResultAreaCallArg(const ResultType& resultType, + CallCompileState* call) { + if (inDeadCode()) { + return true; + } + ABIResultIter iter(resultType); + while (!iter.done() && iter.cur().inRegister()) { + iter.next(); + } + if (iter.done()) { + // No stack results. + return true; + } + + auto* stackResultArea = MWasmStackResultArea::New(alloc()); + if (!stackResultArea) { + return false; + } + if (!stackResultArea->init(alloc(), iter.remaining())) { + return false; + } + for (uint32_t base = iter.index(); !iter.done(); iter.next()) { + MWasmStackResultArea::StackResult loc(iter.cur().stackOffset(), + iter.cur().type().toMIRType()); + stackResultArea->initResult(iter.index() - base, loc); + } + curBlock_->add(stackResultArea); + if (!passArg(stackResultArea, MIRType::Pointer, call)) { + return false; + } + call->stackResultArea_ = stackResultArea; + return true; + } + + [[nodiscard]] bool finishCall(CallCompileState* call) { + if (inDeadCode()) { + return true; + } + + if (!call->regArgs_.append( + MWasmCallBase::Arg(AnyRegister(InstanceReg), instancePointer_))) { + return false; + } + + uint32_t stackBytes = call->abi_.stackBytesConsumedSoFar(); + + maxStackArgBytes_ = std::max(maxStackArgBytes_, stackBytes); + return true; + } + + // Wrappers for creating various kinds of calls. + + [[nodiscard]] bool collectUnaryCallResult(MIRType type, + MDefinition** result) { + MInstruction* def; + switch (type) { + case MIRType::Int32: + def = MWasmRegisterResult::New(alloc(), MIRType::Int32, ReturnReg); + break; + case MIRType::Int64: + def = MWasmRegister64Result::New(alloc(), ReturnReg64); + break; + case MIRType::Float32: + def = MWasmFloatRegisterResult::New(alloc(), type, ReturnFloat32Reg); + break; + case MIRType::Double: + def = MWasmFloatRegisterResult::New(alloc(), type, ReturnDoubleReg); + break; +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: + def = MWasmFloatRegisterResult::New(alloc(), type, ReturnSimd128Reg); + break; +#endif + case MIRType::RefOrNull: + def = MWasmRegisterResult::New(alloc(), MIRType::RefOrNull, ReturnReg); + break; + default: + MOZ_CRASH("unexpected MIRType result for builtin call"); + } + + if (!def) { + return false; + } + + curBlock_->add(def); + *result = def; + + return true; + } + + [[nodiscard]] bool collectCallResults(const ResultType& type, + MWasmStackResultArea* stackResultArea, + DefVector* results) { + if (!results->reserve(type.length())) { + return false; + } + + // The result iterator goes in the order in which results would be popped + // off; we want the order in which they would be pushed. + ABIResultIter iter(type); + uint32_t stackResultCount = 0; + while (!iter.done()) { + if (iter.cur().onStack()) { + stackResultCount++; + } + iter.next(); + } + + for (iter.switchToPrev(); !iter.done(); iter.prev()) { + if (!mirGen().ensureBallast()) { + return false; + } + const ABIResult& result = iter.cur(); + MInstruction* def; + if (result.inRegister()) { + switch (result.type().kind()) { + case wasm::ValType::I32: + def = + MWasmRegisterResult::New(alloc(), MIRType::Int32, result.gpr()); + break; + case wasm::ValType::I64: + def = MWasmRegister64Result::New(alloc(), result.gpr64()); + break; + case wasm::ValType::F32: + def = MWasmFloatRegisterResult::New(alloc(), MIRType::Float32, + result.fpr()); + break; + case wasm::ValType::F64: + def = MWasmFloatRegisterResult::New(alloc(), MIRType::Double, + result.fpr()); + break; + case wasm::ValType::Ref: + def = MWasmRegisterResult::New(alloc(), MIRType::RefOrNull, + result.gpr()); + break; + case wasm::ValType::V128: +#ifdef ENABLE_WASM_SIMD + def = MWasmFloatRegisterResult::New(alloc(), MIRType::Simd128, + result.fpr()); +#else + return this->iter().fail("Ion has no SIMD support yet"); +#endif + } + } else { + MOZ_ASSERT(stackResultArea); + MOZ_ASSERT(stackResultCount); + uint32_t idx = --stackResultCount; + def = MWasmStackResult::New(alloc(), stackResultArea, idx); + } + + if (!def) { + return false; + } + curBlock_->add(def); + results->infallibleAppend(def); + } + + MOZ_ASSERT(results->length() == type.length()); + + return true; + } + + [[nodiscard]] bool catchableCall(const CallSiteDesc& desc, + const CalleeDesc& callee, + const MWasmCallBase::Args& args, + const ArgTypeVector& argTypes, + MDefinition* indexOrRef = nullptr) { + MWasmCallTryDesc tryDesc; + if (!beginTryCall(&tryDesc)) { + return false; + } + + MInstruction* ins; + if (tryDesc.inTry) { + ins = MWasmCallCatchable::New(alloc(), desc, callee, args, + StackArgAreaSizeUnaligned(argTypes), + tryDesc, indexOrRef); + } else { + ins = MWasmCallUncatchable::New(alloc(), desc, callee, args, + StackArgAreaSizeUnaligned(argTypes), + indexOrRef); + } + if (!ins) { + return false; + } + curBlock_->add(ins); + + return finishTryCall(&tryDesc); + } + + [[nodiscard]] bool callDirect(const FuncType& funcType, uint32_t funcIndex, + uint32_t lineOrBytecode, + const CallCompileState& call, + DefVector* results) { + if (inDeadCode()) { + return true; + } + + CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Func); + ResultType resultType = ResultType::Vector(funcType.results()); + auto callee = CalleeDesc::function(funcIndex); + ArgTypeVector args(funcType); + + if (!catchableCall(desc, callee, call.regArgs_, args)) { + return false; + } + return collectCallResults(resultType, call.stackResultArea_, results); + } + + [[nodiscard]] bool callIndirect(uint32_t funcTypeIndex, uint32_t tableIndex, + MDefinition* index, uint32_t lineOrBytecode, + const CallCompileState& call, + DefVector* results) { + if (inDeadCode()) { + return true; + } + + const FuncType& funcType = (*moduleEnv_.types)[funcTypeIndex].funcType(); + CallIndirectId callIndirectId = + CallIndirectId::forFuncType(moduleEnv_, funcTypeIndex); + + CalleeDesc callee; + if (moduleEnv_.isAsmJS()) { + MOZ_ASSERT(tableIndex == 0); + MOZ_ASSERT(callIndirectId.kind() == CallIndirectIdKind::AsmJS); + uint32_t tableIndex = moduleEnv_.asmJSSigToTableIndex[funcTypeIndex]; + const TableDesc& table = moduleEnv_.tables[tableIndex]; + MOZ_ASSERT(IsPowerOfTwo(table.initialLength)); + + MDefinition* mask = constantI32(int32_t(table.initialLength - 1)); + MBitAnd* maskedIndex = MBitAnd::New(alloc(), index, mask, MIRType::Int32); + curBlock_->add(maskedIndex); + + index = maskedIndex; + callee = CalleeDesc::asmJSTable(moduleEnv_, tableIndex); + } else { + MOZ_ASSERT(callIndirectId.kind() != CallIndirectIdKind::AsmJS); + const TableDesc& table = moduleEnv_.tables[tableIndex]; + callee = + CalleeDesc::wasmTable(moduleEnv_, table, tableIndex, callIndirectId); + } + + CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Indirect); + ArgTypeVector args(funcType); + ResultType resultType = ResultType::Vector(funcType.results()); + + if (!catchableCall(desc, callee, call.regArgs_, args, index)) { + return false; + } + return collectCallResults(resultType, call.stackResultArea_, results); + } + + [[nodiscard]] bool callImport(unsigned instanceDataOffset, + uint32_t lineOrBytecode, + const CallCompileState& call, + const FuncType& funcType, DefVector* results) { + if (inDeadCode()) { + return true; + } + + CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Import); + auto callee = CalleeDesc::import(instanceDataOffset); + ArgTypeVector args(funcType); + ResultType resultType = ResultType::Vector(funcType.results()); + + if (!catchableCall(desc, callee, call.regArgs_, args)) { + return false; + } + return collectCallResults(resultType, call.stackResultArea_, results); + } + + [[nodiscard]] bool builtinCall(const SymbolicAddressSignature& builtin, + uint32_t lineOrBytecode, + const CallCompileState& call, + MDefinition** def) { + if (inDeadCode()) { + *def = nullptr; + return true; + } + + MOZ_ASSERT(builtin.failureMode == FailureMode::Infallible); + + CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Symbolic); + auto callee = CalleeDesc::builtin(builtin.identity); + auto* ins = MWasmCallUncatchable::New(alloc(), desc, callee, call.regArgs_, + StackArgAreaSizeUnaligned(builtin)); + if (!ins) { + return false; + } + + curBlock_->add(ins); + + return collectUnaryCallResult(builtin.retType, def); + } + + [[nodiscard]] bool builtinInstanceMethodCall( + const SymbolicAddressSignature& builtin, uint32_t lineOrBytecode, + const CallCompileState& call, MDefinition** def = nullptr) { + MOZ_ASSERT_IF(!def, builtin.retType == MIRType::None); + if (inDeadCode()) { + if (def) { + *def = nullptr; + } + return true; + } + + CallSiteDesc desc(lineOrBytecode, CallSiteDesc::Symbolic); + auto* ins = MWasmCallUncatchable::NewBuiltinInstanceMethodCall( + alloc(), desc, builtin.identity, builtin.failureMode, call.instanceArg_, + call.regArgs_, StackArgAreaSizeUnaligned(builtin)); + if (!ins) { + return false; + } + + curBlock_->add(ins); + + return def ? collectUnaryCallResult(builtin.retType, def) : true; + } + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + [[nodiscard]] bool callRef(const FuncType& funcType, MDefinition* ref, + uint32_t lineOrBytecode, + const CallCompileState& call, DefVector* results) { + if (inDeadCode()) { + return true; + } + + CalleeDesc callee = CalleeDesc::wasmFuncRef(); + + CallSiteDesc desc(lineOrBytecode, CallSiteDesc::FuncRef); + ArgTypeVector args(funcType); + ResultType resultType = ResultType::Vector(funcType.results()); + + if (!catchableCall(desc, callee, call.regArgs_, args, ref)) { + return false; + } + return collectCallResults(resultType, call.stackResultArea_, results); + } + +#endif // ENABLE_WASM_FUNCTION_REFERENCES + + /*********************************************** Control flow generation */ + + inline bool inDeadCode() const { return curBlock_ == nullptr; } + + [[nodiscard]] bool returnValues(const DefVector& values) { + if (inDeadCode()) { + return true; + } + + if (values.empty()) { + curBlock_->end(MWasmReturnVoid::New(alloc(), instancePointer_)); + } else { + ResultType resultType = ResultType::Vector(funcType().results()); + ABIResultIter iter(resultType); + // Switch to iterate in FIFO order instead of the default LIFO. + while (!iter.done()) { + iter.next(); + } + iter.switchToPrev(); + for (uint32_t i = 0; !iter.done(); iter.prev(), i++) { + if (!mirGen().ensureBallast()) { + return false; + } + const ABIResult& result = iter.cur(); + if (result.onStack()) { + MOZ_ASSERT(iter.remaining() > 1); + if (result.type().isRefRepr()) { + auto* store = MWasmStoreRef::New( + alloc(), instancePointer_, stackResultPointer_, + result.stackOffset(), values[i], AliasSet::WasmStackResult, + WasmPreBarrierKind::None); + curBlock_->add(store); + } else { + auto* store = MWasmStoreStackResult::New( + alloc(), stackResultPointer_, result.stackOffset(), values[i]); + curBlock_->add(store); + } + } else { + MOZ_ASSERT(iter.remaining() == 1); + MOZ_ASSERT(i + 1 == values.length()); + curBlock_->end( + MWasmReturn::New(alloc(), values[i], instancePointer_)); + } + } + } + curBlock_ = nullptr; + return true; + } + + void unreachableTrap() { + if (inDeadCode()) { + return; + } + + auto* ins = + MWasmTrap::New(alloc(), wasm::Trap::Unreachable, bytecodeOffset()); + curBlock_->end(ins); + curBlock_ = nullptr; + } + + private: + static uint32_t numPushed(MBasicBlock* block) { + return block->stackDepth() - block->info().firstStackSlot(); + } + + public: + [[nodiscard]] bool pushDefs(const DefVector& defs) { + if (inDeadCode()) { + return true; + } + MOZ_ASSERT(numPushed(curBlock_) == 0); + if (!curBlock_->ensureHasSlots(defs.length())) { + return false; + } + for (MDefinition* def : defs) { + MOZ_ASSERT(def->type() != MIRType::None); + curBlock_->push(def); + } + return true; + } + + [[nodiscard]] bool popPushedDefs(DefVector* defs) { + size_t n = numPushed(curBlock_); + if (!defs->resizeUninitialized(n)) { + return false; + } + for (; n > 0; n--) { + MDefinition* def = curBlock_->pop(); + MOZ_ASSERT(def->type() != MIRType::Value); + (*defs)[n - 1] = def; + } + return true; + } + + private: + [[nodiscard]] bool addJoinPredecessor(const DefVector& defs, + MBasicBlock** joinPred) { + *joinPred = curBlock_; + if (inDeadCode()) { + return true; + } + return pushDefs(defs); + } + + public: + [[nodiscard]] bool branchAndStartThen(MDefinition* cond, + MBasicBlock** elseBlock) { + if (inDeadCode()) { + *elseBlock = nullptr; + } else { + MBasicBlock* thenBlock; + if (!newBlock(curBlock_, &thenBlock)) { + return false; + } + if (!newBlock(curBlock_, elseBlock)) { + return false; + } + + curBlock_->end(MTest::New(alloc(), cond, thenBlock, *elseBlock)); + + curBlock_ = thenBlock; + mirGraph().moveBlockToEnd(curBlock_); + } + + return startBlock(); + } + + [[nodiscard]] bool switchToElse(MBasicBlock* elseBlock, + MBasicBlock** thenJoinPred) { + DefVector values; + if (!finishBlock(&values)) { + return false; + } + + if (!elseBlock) { + *thenJoinPred = nullptr; + } else { + if (!addJoinPredecessor(values, thenJoinPred)) { + return false; + } + + curBlock_ = elseBlock; + mirGraph().moveBlockToEnd(curBlock_); + } + + return startBlock(); + } + + [[nodiscard]] bool joinIfElse(MBasicBlock* thenJoinPred, DefVector* defs) { + DefVector values; + if (!finishBlock(&values)) { + return false; + } + + if (!thenJoinPred && inDeadCode()) { + return true; + } + + MBasicBlock* elseJoinPred; + if (!addJoinPredecessor(values, &elseJoinPred)) { + return false; + } + + mozilla::Array blocks; + size_t numJoinPreds = 0; + if (thenJoinPred) { + blocks[numJoinPreds++] = thenJoinPred; + } + if (elseJoinPred) { + blocks[numJoinPreds++] = elseJoinPred; + } + + if (numJoinPreds == 0) { + return true; + } + + MBasicBlock* join; + if (!goToNewBlock(blocks[0], &join)) { + return false; + } + for (size_t i = 1; i < numJoinPreds; ++i) { + if (!goToExistingBlock(blocks[i], join)) { + return false; + } + } + + curBlock_ = join; + return popPushedDefs(defs); + } + + [[nodiscard]] bool startBlock() { + MOZ_ASSERT_IF(blockDepth_ < blockPatches_.length(), + blockPatches_[blockDepth_].empty()); + blockDepth_++; + return true; + } + + [[nodiscard]] bool finishBlock(DefVector* defs) { + MOZ_ASSERT(blockDepth_); + uint32_t topLabel = --blockDepth_; + return bindBranches(topLabel, defs); + } + + [[nodiscard]] bool startLoop(MBasicBlock** loopHeader, size_t paramCount) { + *loopHeader = nullptr; + + blockDepth_++; + loopDepth_++; + + if (inDeadCode()) { + return true; + } + + // Create the loop header. + MOZ_ASSERT(curBlock_->loopDepth() == loopDepth_ - 1); + *loopHeader = MBasicBlock::New(mirGraph(), info(), curBlock_, + MBasicBlock::PENDING_LOOP_HEADER); + if (!*loopHeader) { + return false; + } + + (*loopHeader)->setLoopDepth(loopDepth_); + mirGraph().addBlock(*loopHeader); + curBlock_->end(MGoto::New(alloc(), *loopHeader)); + + DefVector loopParams; + if (!iter().getResults(paramCount, &loopParams)) { + return false; + } + for (size_t i = 0; i < paramCount; i++) { + MPhi* phi = MPhi::New(alloc(), loopParams[i]->type()); + if (!phi) { + return false; + } + if (!phi->reserveLength(2)) { + return false; + } + (*loopHeader)->addPhi(phi); + phi->addInput(loopParams[i]); + loopParams[i] = phi; + } + iter().setResults(paramCount, loopParams); + + MBasicBlock* body; + if (!goToNewBlock(*loopHeader, &body)) { + return false; + } + curBlock_ = body; + return true; + } + + private: + void fixupRedundantPhis(MBasicBlock* b) { + for (size_t i = 0, depth = b->stackDepth(); i < depth; i++) { + MDefinition* def = b->getSlot(i); + if (def->isUnused()) { + b->setSlot(i, def->toPhi()->getOperand(0)); + } + } + } + + [[nodiscard]] bool setLoopBackedge(MBasicBlock* loopEntry, + MBasicBlock* loopBody, + MBasicBlock* backedge, size_t paramCount) { + if (!loopEntry->setBackedgeWasm(backedge, paramCount)) { + return false; + } + + // Flag all redundant phis as unused. + for (MPhiIterator phi = loopEntry->phisBegin(); phi != loopEntry->phisEnd(); + phi++) { + MOZ_ASSERT(phi->numOperands() == 2); + if (phi->getOperand(0) == phi->getOperand(1)) { + phi->setUnused(); + } + } + + // Fix up phis stored in the slots Vector of pending blocks. + for (ControlFlowPatchVector& patches : blockPatches_) { + for (ControlFlowPatch& p : patches) { + MBasicBlock* block = p.ins->block(); + if (block->loopDepth() >= loopEntry->loopDepth()) { + fixupRedundantPhis(block); + } + } + } + + // The loop body, if any, might be referencing recycled phis too. + if (loopBody) { + fixupRedundantPhis(loopBody); + } + + // Pending jumps to an enclosing try-catch may reference the recycled phis. + // We have to search above all enclosing try blocks, as a delegate may move + // patches around. + for (uint32_t depth = 0; depth < iter().controlStackDepth(); depth++) { + LabelKind kind = iter().controlKind(depth); + if (kind != LabelKind::Try && kind != LabelKind::Body) { + continue; + } + Control& control = iter().controlItem(depth); + for (MControlInstruction* patch : control.tryPadPatches) { + MBasicBlock* block = patch->block(); + if (block->loopDepth() >= loopEntry->loopDepth()) { + fixupRedundantPhis(block); + } + } + } + + // Discard redundant phis and add to the free list. + for (MPhiIterator phi = loopEntry->phisBegin(); + phi != loopEntry->phisEnd();) { + MPhi* entryDef = *phi++; + if (!entryDef->isUnused()) { + continue; + } + + entryDef->justReplaceAllUsesWith(entryDef->getOperand(0)); + loopEntry->discardPhi(entryDef); + mirGraph().addPhiToFreeList(entryDef); + } + + return true; + } + + public: + [[nodiscard]] bool closeLoop(MBasicBlock* loopHeader, + DefVector* loopResults) { + MOZ_ASSERT(blockDepth_ >= 1); + MOZ_ASSERT(loopDepth_); + + uint32_t headerLabel = blockDepth_ - 1; + + if (!loopHeader) { + MOZ_ASSERT(inDeadCode()); + MOZ_ASSERT(headerLabel >= blockPatches_.length() || + blockPatches_[headerLabel].empty()); + blockDepth_--; + loopDepth_--; + return true; + } + + // Op::Loop doesn't have an implicit backedge so temporarily set + // aside the end of the loop body to bind backedges. + MBasicBlock* loopBody = curBlock_; + curBlock_ = nullptr; + + // As explained in bug 1253544, Ion apparently has an invariant that + // there is only one backedge to loop headers. To handle wasm's ability + // to have multiple backedges to the same loop header, we bind all those + // branches as forward jumps to a single backward jump. This is + // unfortunate but the optimizer is able to fold these into single jumps + // to backedges. + DefVector backedgeValues; + if (!bindBranches(headerLabel, &backedgeValues)) { + return false; + } + + MOZ_ASSERT(loopHeader->loopDepth() == loopDepth_); + + if (curBlock_) { + // We're on the loop backedge block, created by bindBranches. + for (size_t i = 0, n = numPushed(curBlock_); i != n; i++) { + curBlock_->pop(); + } + + if (!pushDefs(backedgeValues)) { + return false; + } + + MOZ_ASSERT(curBlock_->loopDepth() == loopDepth_); + curBlock_->end(MGoto::New(alloc(), loopHeader)); + if (!setLoopBackedge(loopHeader, loopBody, curBlock_, + backedgeValues.length())) { + return false; + } + } + + curBlock_ = loopBody; + + loopDepth_--; + + // If the loop depth still at the inner loop body, correct it. + if (curBlock_ && curBlock_->loopDepth() != loopDepth_) { + MBasicBlock* out; + if (!goToNewBlock(curBlock_, &out)) { + return false; + } + curBlock_ = out; + } + + blockDepth_ -= 1; + return inDeadCode() || popPushedDefs(loopResults); + } + + [[nodiscard]] bool addControlFlowPatch(MControlInstruction* ins, + uint32_t relative, uint32_t index) { + MOZ_ASSERT(relative < blockDepth_); + uint32_t absolute = blockDepth_ - 1 - relative; + + if (absolute >= blockPatches_.length() && + !blockPatches_.resize(absolute + 1)) { + return false; + } + + return blockPatches_[absolute].append(ControlFlowPatch(ins, index)); + } + + [[nodiscard]] bool br(uint32_t relativeDepth, const DefVector& values) { + if (inDeadCode()) { + return true; + } + + MGoto* jump = MGoto::New(alloc()); + if (!addControlFlowPatch(jump, relativeDepth, MGoto::TargetIndex)) { + return false; + } + + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(jump); + curBlock_ = nullptr; + return true; + } + + [[nodiscard]] bool brIf(uint32_t relativeDepth, const DefVector& values, + MDefinition* condition) { + if (inDeadCode()) { + return true; + } + + MBasicBlock* joinBlock = nullptr; + if (!newBlock(curBlock_, &joinBlock)) { + return false; + } + + MTest* test = MTest::New(alloc(), condition, nullptr, joinBlock); + if (!addControlFlowPatch(test, relativeDepth, MTest::TrueBranchIndex)) { + return false; + } + + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(test); + curBlock_ = joinBlock; + return true; + } + + [[nodiscard]] bool brTable(MDefinition* operand, uint32_t defaultDepth, + const Uint32Vector& depths, + const DefVector& values) { + if (inDeadCode()) { + return true; + } + + size_t numCases = depths.length(); + MOZ_ASSERT(numCases <= INT32_MAX); + MOZ_ASSERT(numCases); + + MTableSwitch* table = + MTableSwitch::New(alloc(), operand, 0, int32_t(numCases - 1)); + + size_t defaultIndex; + if (!table->addDefault(nullptr, &defaultIndex)) { + return false; + } + if (!addControlFlowPatch(table, defaultDepth, defaultIndex)) { + return false; + } + + using IndexToCaseMap = + HashMap, SystemAllocPolicy>; + + IndexToCaseMap indexToCase; + if (!indexToCase.put(defaultDepth, defaultIndex)) { + return false; + } + + for (size_t i = 0; i < numCases; i++) { + if (!mirGen_.ensureBallast()) { + return false; + } + + uint32_t depth = depths[i]; + + size_t caseIndex; + IndexToCaseMap::AddPtr p = indexToCase.lookupForAdd(depth); + if (!p) { + if (!table->addSuccessor(nullptr, &caseIndex)) { + return false; + } + if (!addControlFlowPatch(table, depth, caseIndex)) { + return false; + } + if (!indexToCase.add(p, depth, caseIndex)) { + return false; + } + } else { + caseIndex = p->value(); + } + + if (!table->addCase(caseIndex)) { + return false; + } + } + + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(table); + curBlock_ = nullptr; + + return true; + } + + /********************************************************** Exceptions ***/ + + bool inTryBlock(uint32_t* relativeDepth) { + return iter().controlFindInnermost(LabelKind::Try, relativeDepth); + } + + bool inTryCode() { + uint32_t relativeDepth; + return inTryBlock(&relativeDepth); + } + + MDefinition* loadTag(uint32_t tagIndex) { + MWasmLoadInstanceDataField* tag = MWasmLoadInstanceDataField::New( + alloc(), MIRType::RefOrNull, + moduleEnv_.offsetOfTagInstanceData(tagIndex), true, instancePointer_); + curBlock_->add(tag); + return tag; + } + + void loadPendingExceptionState(MInstruction** exception, MInstruction** tag) { + *exception = MWasmLoadInstance::New( + alloc(), instancePointer_, wasm::Instance::offsetOfPendingException(), + MIRType::RefOrNull, AliasSet::Load(AliasSet::WasmPendingException)); + curBlock_->add(*exception); + + *tag = MWasmLoadInstance::New( + alloc(), instancePointer_, + wasm::Instance::offsetOfPendingExceptionTag(), MIRType::RefOrNull, + AliasSet::Load(AliasSet::WasmPendingException)); + curBlock_->add(*tag); + } + + [[nodiscard]] bool setPendingExceptionState(MDefinition* exception, + MDefinition* tag) { + // Set the pending exception object + auto* exceptionAddr = MWasmDerivedPointer::New( + alloc(), instancePointer_, Instance::offsetOfPendingException()); + curBlock_->add(exceptionAddr); + auto* setException = MWasmStoreRef::New( + alloc(), instancePointer_, exceptionAddr, /*valueOffset=*/0, exception, + AliasSet::WasmPendingException, WasmPreBarrierKind::Normal); + curBlock_->add(setException); + if (!postBarrierPrecise(/*lineOrBytecode=*/0, exceptionAddr, exception)) { + return false; + } + + // Set the pending exception tag object + auto* exceptionTagAddr = MWasmDerivedPointer::New( + alloc(), instancePointer_, Instance::offsetOfPendingExceptionTag()); + curBlock_->add(exceptionTagAddr); + auto* setExceptionTag = MWasmStoreRef::New( + alloc(), instancePointer_, exceptionTagAddr, /*valueOffset=*/0, tag, + AliasSet::WasmPendingException, WasmPreBarrierKind::Normal); + curBlock_->add(setExceptionTag); + return postBarrierPrecise(/*lineOrBytecode=*/0, exceptionTagAddr, tag); + } + + [[nodiscard]] bool addPadPatch(MControlInstruction* ins, + size_t relativeTryDepth) { + Control& tryControl = iter().controlItem(relativeTryDepth); + ControlInstructionVector& padPatches = tryControl.tryPadPatches; + return padPatches.emplaceBack(ins); + } + + [[nodiscard]] bool endWithPadPatch(uint32_t relativeTryDepth) { + MGoto* jumpToLandingPad = MGoto::New(alloc()); + curBlock_->end(jumpToLandingPad); + return addPadPatch(jumpToLandingPad, relativeTryDepth); + } + + [[nodiscard]] bool delegatePadPatches(const ControlInstructionVector& patches, + uint32_t relativeDepth) { + if (patches.empty()) { + return true; + } + + // Find where we are delegating the pad patches to. + uint32_t targetRelativeDepth; + if (!iter().controlFindInnermostFrom(LabelKind::Try, relativeDepth, + &targetRelativeDepth)) { + MOZ_ASSERT(relativeDepth <= blockDepth_ - 1); + targetRelativeDepth = blockDepth_ - 1; + } + // Append the delegate's pad patches to the target's. + for (MControlInstruction* ins : patches) { + if (!addPadPatch(ins, targetRelativeDepth)) { + return false; + } + } + return true; + } + + [[nodiscard]] bool beginTryCall(MWasmCallTryDesc* call) { + call->inTry = inTryBlock(&call->relativeTryDepth); + if (!call->inTry) { + return true; + } + // Allocate a try note + if (!tryNotes_.append(wasm::TryNote())) { + return false; + } + call->tryNoteIndex = tryNotes_.length() - 1; + // Allocate blocks for fallthrough and exceptions + return newBlock(curBlock_, &call->fallthroughBlock) && + newBlock(curBlock_, &call->prePadBlock); + } + + [[nodiscard]] bool finishTryCall(MWasmCallTryDesc* call) { + if (!call->inTry) { + return true; + } + + // Switch to the prePadBlock + MBasicBlock* callBlock = curBlock_; + curBlock_ = call->prePadBlock; + + // Mark this as the landing pad for the call + curBlock_->add( + MWasmCallLandingPrePad::New(alloc(), callBlock, call->tryNoteIndex)); + + // End with a pending jump to the landing pad + if (!endWithPadPatch(call->relativeTryDepth)) { + return false; + } + + // Compilation continues in the fallthroughBlock. + curBlock_ = call->fallthroughBlock; + return true; + } + + // Create a landing pad for a try block if there are any throwing + // instructions. + [[nodiscard]] bool createTryLandingPadIfNeeded(Control& control, + MBasicBlock** landingPad) { + // If there are no pad-patches for this try control, it means there are no + // instructions in the try code that could throw an exception. In this + // case, all the catches are dead code, and the try code ends up equivalent + // to a plain wasm block. + ControlInstructionVector& patches = control.tryPadPatches; + if (patches.empty()) { + *landingPad = nullptr; + return true; + } + + // Otherwise, if there are (pad-) branches from places in the try code that + // may throw an exception, bind these branches to a new landing pad + // block. This is done similarly to what is done in bindBranches. + MControlInstruction* ins = patches[0]; + MBasicBlock* pred = ins->block(); + if (!newBlock(pred, landingPad)) { + return false; + } + ins->replaceSuccessor(0, *landingPad); + for (size_t i = 1; i < patches.length(); i++) { + ins = patches[i]; + pred = ins->block(); + if (!(*landingPad)->addPredecessor(alloc(), pred)) { + return false; + } + ins->replaceSuccessor(0, *landingPad); + } + + // Set up the slots in the landing pad block. + if (!setupLandingPadSlots(*landingPad)) { + return false; + } + + // Clear the now bound pad patches. + patches.clear(); + return true; + } + + // Consume the pending exception state from instance, and set up the slots + // of the landing pad with the exception state. + [[nodiscard]] bool setupLandingPadSlots(MBasicBlock* landingPad) { + MBasicBlock* prevBlock = curBlock_; + curBlock_ = landingPad; + + // Load the pending exception and tag + MInstruction* exception; + MInstruction* tag; + loadPendingExceptionState(&exception, &tag); + + // Clear the pending exception and tag + auto* null = constantNullRef(); + if (!setPendingExceptionState(null, null)) { + return false; + } + + // Push the exception and its tag on the stack to make them available + // to the landing pad blocks. + if (!landingPad->ensureHasSlots(2)) { + return false; + } + landingPad->push(exception); + landingPad->push(tag); + + curBlock_ = prevBlock; + return true; + } + + [[nodiscard]] bool startTry(MBasicBlock** curBlock) { + *curBlock = curBlock_; + return startBlock(); + } + + [[nodiscard]] bool joinTryOrCatchBlock(Control& control) { + // If the try or catch block ended with dead code, there is no need to + // do any control flow join. + if (inDeadCode()) { + return true; + } + + // This is a split path which we'll need to join later, using a control + // flow patch. + MOZ_ASSERT(!curBlock_->hasLastIns()); + MGoto* jump = MGoto::New(alloc()); + if (!addControlFlowPatch(jump, 0, MGoto::TargetIndex)) { + return false; + } + + // Finish the current block with the control flow patch instruction. + curBlock_->end(jump); + return true; + } + + // Finish the previous block (either a try or catch block) and then setup a + // new catch block. + [[nodiscard]] bool switchToCatch(Control& control, const LabelKind& fromKind, + uint32_t tagIndex) { + // If there is no control block, then either: + // - the entry of the try block is dead code, or + // - there is no landing pad for the try-catch. + // In either case, any catch will be dead code. + if (!control.block) { + MOZ_ASSERT(inDeadCode()); + return true; + } + + // Join the previous try or catch block with a patch to the future join of + // the whole try-catch block. + if (!joinTryOrCatchBlock(control)) { + return false; + } + + // If we are switching from the try block, create the landing pad. This is + // guaranteed to happen once and only once before processing catch blocks. + if (fromKind == LabelKind::Try) { + MBasicBlock* padBlock = nullptr; + if (!createTryLandingPadIfNeeded(control, &padBlock)) { + return false; + } + // Set the control block for this try-catch to the landing pad. + control.block = padBlock; + } + + // If there is no landing pad, then this and following catches are dead + // code. + if (!control.block) { + curBlock_ = nullptr; + return true; + } + + // Switch to the landing pad. + curBlock_ = control.block; + + // Handle a catch_all by immediately jumping to a new block. We require a + // new block (as opposed to just emitting the catch_all code in the current + // block) because rethrow requires the exception/tag to be present in the + // landing pad's slots, while the catch_all block must not have the + // exception/tag in slots. + if (tagIndex == CatchAllIndex) { + MBasicBlock* catchAllBlock = nullptr; + if (!goToNewBlock(curBlock_, &catchAllBlock)) { + return false; + } + // Compilation will continue in the catch_all block. + curBlock_ = catchAllBlock; + // Remove the tag and exception slots from the block, they are no + // longer necessary. + curBlock_->pop(); + curBlock_->pop(); + return true; + } + + // Handle a tagged catch by doing a compare and branch on the tag index, + // jumping to a catch block if they match, or else to a fallthrough block + // to continue the landing pad. + MBasicBlock* catchBlock = nullptr; + MBasicBlock* fallthroughBlock = nullptr; + if (!newBlock(curBlock_, &catchBlock) || + !newBlock(curBlock_, &fallthroughBlock)) { + return false; + } + + // Get the exception and its tag from the slots we pushed when adding + // control flow patches. + MDefinition* exceptionTag = curBlock_->pop(); + MDefinition* exception = curBlock_->pop(); + + // Branch to the catch block if the exception's tag matches this catch + // block's tag. + MDefinition* catchTag = loadTag(tagIndex); + MDefinition* matchesCatchTag = + compare(exceptionTag, catchTag, JSOp::Eq, MCompare::Compare_RefOrNull); + curBlock_->end( + MTest::New(alloc(), matchesCatchTag, catchBlock, fallthroughBlock)); + + // The landing pad will continue in the fallthrough block + control.block = fallthroughBlock; + + // Set up the catch block by extracting the values from the exception + // object. + curBlock_ = catchBlock; + + // Remove the tag and exception slots from the block, they are no + // longer necessary. + curBlock_->pop(); + curBlock_->pop(); + + // Extract the exception values for the catch block + DefVector values; + if (!loadExceptionValues(exception, tagIndex, &values)) { + return false; + } + iter().setResults(values.length(), values); + return true; + } + + [[nodiscard]] bool loadExceptionValues(MDefinition* exception, + uint32_t tagIndex, DefVector* values) { + SharedTagType tagType = moduleEnv().tags[tagIndex].type; + const ValTypeVector& params = tagType->argTypes_; + const TagOffsetVector& offsets = tagType->argOffsets_; + + // Get the data pointer from the exception object + auto* data = MWasmLoadField::New( + alloc(), exception, WasmExceptionObject::offsetOfData(), + MIRType::Pointer, MWideningOp::None, AliasSet::Load(AliasSet::Any)); + if (!data) { + return false; + } + curBlock_->add(data); + + // Presize the values vector to the number of params + if (!values->reserve(params.length())) { + return false; + } + + // Load each value from the data pointer + for (size_t i = 0; i < params.length(); i++) { + if (!mirGen_.ensureBallast()) { + return false; + } + auto* load = MWasmLoadFieldKA::New( + alloc(), exception, data, offsets[i], params[i].toMIRType(), + MWideningOp::None, AliasSet::Load(AliasSet::Any)); + if (!load || !values->append(load)) { + return false; + } + curBlock_->add(load); + } + return true; + } + + [[nodiscard]] bool finishTryCatch(LabelKind kind, Control& control, + DefVector* defs) { + switch (kind) { + case LabelKind::Try: { + // This is a catchless try, we must delegate all throwing instructions + // to the nearest enclosing try block if one exists, or else to the + // body block which will handle it in emitBodyDelegateThrowPad. We + // specify a relativeDepth of '1' to delegate outside of the still + // active try block. + uint32_t relativeDepth = 1; + if (!delegatePadPatches(control.tryPadPatches, relativeDepth)) { + return false; + } + break; + } + case LabelKind::Catch: { + // This is a try without a catch_all, we must have a rethrow at the end + // of the landing pad (if any). + MBasicBlock* padBlock = control.block; + if (padBlock) { + MBasicBlock* prevBlock = curBlock_; + curBlock_ = padBlock; + MDefinition* tag = curBlock_->pop(); + MDefinition* exception = curBlock_->pop(); + if (!throwFrom(exception, tag)) { + return false; + } + curBlock_ = prevBlock; + } + break; + } + case LabelKind::CatchAll: + // This is a try with a catch_all, and requires no special handling. + break; + default: + MOZ_CRASH(); + } + + // Finish the block, joining the try and catch blocks + return finishBlock(defs); + } + + [[nodiscard]] bool emitBodyDelegateThrowPad(Control& control) { + // Create a landing pad for any throwing instructions + MBasicBlock* padBlock; + if (!createTryLandingPadIfNeeded(control, &padBlock)) { + return false; + } + + // If no landing pad was necessary, then we don't need to do anything here + if (!padBlock) { + return true; + } + + // Switch to the landing pad and rethrow the exception + MBasicBlock* prevBlock = curBlock_; + curBlock_ = padBlock; + MDefinition* tag = curBlock_->pop(); + MDefinition* exception = curBlock_->pop(); + if (!throwFrom(exception, tag)) { + return false; + } + curBlock_ = prevBlock; + return true; + } + + [[nodiscard]] bool emitNewException(MDefinition* tag, + MDefinition** exception) { + return emitInstanceCall1(readBytecodeOffset(), SASigExceptionNew, tag, + exception); + } + + [[nodiscard]] bool emitThrow(uint32_t tagIndex, const DefVector& argValues) { + if (inDeadCode()) { + return true; + } + uint32_t bytecodeOffset = readBytecodeOffset(); + + // Load the tag + MDefinition* tag = loadTag(tagIndex); + if (!tag) { + return false; + } + + // Allocate an exception object + MDefinition* exception; + if (!emitNewException(tag, &exception)) { + return false; + } + + // Load the data pointer from the object + auto* data = MWasmLoadField::New( + alloc(), exception, WasmExceptionObject::offsetOfData(), + MIRType::Pointer, MWideningOp::None, AliasSet::Load(AliasSet::Any)); + if (!data) { + return false; + } + curBlock_->add(data); + + // Store the params into the data pointer + SharedTagType tagType = moduleEnv_.tags[tagIndex].type; + for (size_t i = 0; i < tagType->argOffsets_.length(); i++) { + if (!mirGen_.ensureBallast()) { + return false; + } + ValType type = tagType->argTypes_[i]; + uint32_t offset = tagType->argOffsets_[i]; + + if (!type.isRefRepr()) { + auto* store = MWasmStoreFieldKA::New(alloc(), exception, data, offset, + argValues[i], MNarrowingOp::None, + AliasSet::Store(AliasSet::Any)); + if (!store) { + return false; + } + curBlock_->add(store); + continue; + } + + // Store the new value + auto* store = MWasmStoreFieldRefKA::New( + alloc(), instancePointer_, exception, data, offset, argValues[i], + AliasSet::Store(AliasSet::Any), Nothing(), WasmPreBarrierKind::None); + if (!store) { + return false; + } + curBlock_->add(store); + + // Call the post-write barrier + if (!postBarrier(bytecodeOffset, exception, data, offset, argValues[i])) { + return false; + } + } + + // Throw the exception + return throwFrom(exception, tag); + } + + [[nodiscard]] bool throwFrom(MDefinition* exn, MDefinition* tag) { + if (inDeadCode()) { + return true; + } + + // Check if there is a local catching try control, and if so, then add a + // pad-patch to its tryPadPatches. + uint32_t relativeTryDepth; + if (inTryBlock(&relativeTryDepth)) { + // Set the pending exception state, the landing pad will read from this + if (!setPendingExceptionState(exn, tag)) { + return false; + } + + // End with a pending jump to the landing pad + if (!endWithPadPatch(relativeTryDepth)) { + return false; + } + curBlock_ = nullptr; + return true; + } + + // If there is no surrounding catching block, call an instance method to + // throw the exception. + if (!emitInstanceCall1(readBytecodeOffset(), SASigThrowException, exn)) { + return false; + } + unreachableTrap(); + + curBlock_ = nullptr; + return true; + } + + [[nodiscard]] bool emitRethrow(uint32_t relativeDepth) { + if (inDeadCode()) { + return true; + } + + Control& control = iter().controlItem(relativeDepth); + MBasicBlock* pad = control.block; + MOZ_ASSERT(pad); + MOZ_ASSERT(pad->nslots() > 1); + MOZ_ASSERT(iter().controlKind(relativeDepth) == LabelKind::Catch || + iter().controlKind(relativeDepth) == LabelKind::CatchAll); + + // The exception will always be the last slot in the landing pad. + size_t exnSlotPosition = pad->nslots() - 2; + MDefinition* tag = pad->getSlot(exnSlotPosition + 1); + MDefinition* exception = pad->getSlot(exnSlotPosition); + MOZ_ASSERT(exception->type() == MIRType::RefOrNull && + tag->type() == MIRType::RefOrNull); + return throwFrom(exception, tag); + } + + /*********************************************** Instance call helpers ***/ + + // Do not call this function directly -- it offers no protection against + // mis-counting of arguments. Instead call one of + // ::emitInstanceCall{0,1,2,3,4,5,6}. + // + // Emits a call to the Instance function indicated by `callee`. This is + // assumed to take an Instance pointer as its first argument. The remaining + // args are taken from `args`, which is assumed to hold `numArgs` entries. + // If `result` is non-null, the MDefinition* holding the return value is + // written to `*result`. + [[nodiscard]] bool emitInstanceCallN(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition** args, size_t numArgs, + MDefinition** result = nullptr) { + // Check that the first formal parameter is plausibly an Instance pointer. + MOZ_ASSERT(callee.numArgs > 0); + MOZ_ASSERT(callee.argTypes[0] == MIRType::Pointer); + // Check we agree on the number of args. + MOZ_ASSERT(numArgs + 1 /* the instance pointer */ == callee.numArgs); + // Check we agree on whether a value is returned. + MOZ_ASSERT((result == nullptr) == (callee.retType == MIRType::None)); + + // If we are in dead code, it can happen that some of the `args` entries + // are nullptr, which will look like an OOM to the logic below. So exit + // at this point. `passInstance`, `passArg`, `finishCall` and + // `builtinInstanceMethodCall` all do nothing in dead code, so it's valid + // to exit here. + if (inDeadCode()) { + if (result) { + *result = nullptr; + } + return true; + } + + // Check all args for signs of OOMness before attempting to allocating any + // more memory. + for (size_t i = 0; i < numArgs; i++) { + if (!args[i]) { + if (result) { + *result = nullptr; + } + return false; + } + } + + // Finally, construct the call. + CallCompileState ccsArgs; + if (!passInstance(callee.argTypes[0], &ccsArgs)) { + return false; + } + for (size_t i = 0; i < numArgs; i++) { + if (!passArg(args[i], callee.argTypes[i + 1], &ccsArgs)) { + return false; + } + } + if (!finishCall(&ccsArgs)) { + return false; + } + return builtinInstanceMethodCall(callee, lineOrBytecode, ccsArgs, result); + } + + [[nodiscard]] bool emitInstanceCall0(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition** result = nullptr) { + MDefinition* args[0] = {}; + return emitInstanceCallN(lineOrBytecode, callee, args, 0, result); + } + [[nodiscard]] bool emitInstanceCall1(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition* arg1, + MDefinition** result = nullptr) { + MDefinition* args[1] = {arg1}; + return emitInstanceCallN(lineOrBytecode, callee, args, 1, result); + } + [[nodiscard]] bool emitInstanceCall2(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition* arg1, MDefinition* arg2, + MDefinition** result = nullptr) { + MDefinition* args[2] = {arg1, arg2}; + return emitInstanceCallN(lineOrBytecode, callee, args, 2, result); + } + [[nodiscard]] bool emitInstanceCall3(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition* arg1, MDefinition* arg2, + MDefinition* arg3, + MDefinition** result = nullptr) { + MDefinition* args[3] = {arg1, arg2, arg3}; + return emitInstanceCallN(lineOrBytecode, callee, args, 3, result); + } + [[nodiscard]] bool emitInstanceCall4(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition* arg1, MDefinition* arg2, + MDefinition* arg3, MDefinition* arg4, + MDefinition** result = nullptr) { + MDefinition* args[4] = {arg1, arg2, arg3, arg4}; + return emitInstanceCallN(lineOrBytecode, callee, args, 4, result); + } + [[nodiscard]] bool emitInstanceCall5(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition* arg1, MDefinition* arg2, + MDefinition* arg3, MDefinition* arg4, + MDefinition* arg5, + MDefinition** result = nullptr) { + MDefinition* args[5] = {arg1, arg2, arg3, arg4, arg5}; + return emitInstanceCallN(lineOrBytecode, callee, args, 5, result); + } + [[nodiscard]] bool emitInstanceCall6(uint32_t lineOrBytecode, + const SymbolicAddressSignature& callee, + MDefinition* arg1, MDefinition* arg2, + MDefinition* arg3, MDefinition* arg4, + MDefinition* arg5, MDefinition* arg6, + MDefinition** result = nullptr) { + MDefinition* args[6] = {arg1, arg2, arg3, arg4, arg5, arg6}; + return emitInstanceCallN(lineOrBytecode, callee, args, 6, result); + } + + /******************************** WasmGC: low level load/store helpers ***/ + + // Given a (FieldType, FieldExtension) pair, produce the (MIRType, + // MWideningOp) pair that will give the correct operation for reading the + // value from memory. + static void fieldLoadInfoToMIR(FieldType type, FieldWideningOp wideningOp, + MIRType* mirType, MWideningOp* mirWideningOp) { + switch (type.kind()) { + case FieldType::I8: { + switch (wideningOp) { + case FieldWideningOp::Signed: + *mirType = MIRType::Int32; + *mirWideningOp = MWideningOp::FromS8; + return; + case FieldWideningOp::Unsigned: + *mirType = MIRType::Int32; + *mirWideningOp = MWideningOp::FromU8; + return; + default: + MOZ_CRASH(); + } + } + case FieldType::I16: { + switch (wideningOp) { + case FieldWideningOp::Signed: + *mirType = MIRType::Int32; + *mirWideningOp = MWideningOp::FromS16; + return; + case FieldWideningOp::Unsigned: + *mirType = MIRType::Int32; + *mirWideningOp = MWideningOp::FromU16; + return; + default: + MOZ_CRASH(); + } + } + default: { + switch (wideningOp) { + case FieldWideningOp::None: + *mirType = type.toMIRType(); + *mirWideningOp = MWideningOp::None; + return; + default: + MOZ_CRASH(); + } + } + } + } + + // Given a FieldType, produce the MNarrowingOp required for writing the + // value to memory. + static MNarrowingOp fieldStoreInfoToMIR(FieldType type) { + switch (type.kind()) { + case FieldType::I8: + return MNarrowingOp::To8; + case FieldType::I16: + return MNarrowingOp::To16; + default: + return MNarrowingOp::None; + } + } + + // Generate a write of `value` at address `base + offset`, where `offset` is + // known at JIT time. If the written value is a reftype, the previous value + // at `base + offset` will be retrieved and handed off to the post-write + // barrier. `keepAlive` will be referenced by the instruction so as to hold + // it live (from the GC's point of view). + [[nodiscard]] bool writeGcValueAtBasePlusOffset( + uint32_t lineOrBytecode, FieldType fieldType, MDefinition* keepAlive, + AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base, + uint32_t offset, bool needsTrapInfo, WasmPreBarrierKind preBarrierKind) { + MOZ_ASSERT(aliasBitset != 0); + MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull); + MOZ_ASSERT(fieldType.widenToValType().toMIRType() == value->type()); + MNarrowingOp narrowingOp = fieldStoreInfoToMIR(fieldType); + + if (!fieldType.isRefRepr()) { + MaybeTrapSiteInfo maybeTrap; + if (needsTrapInfo) { + maybeTrap.emplace(getTrapSiteInfo()); + } + auto* store = MWasmStoreFieldKA::New( + alloc(), keepAlive, base, offset, value, narrowingOp, + AliasSet::Store(aliasBitset), maybeTrap); + if (!store) { + return false; + } + curBlock_->add(store); + return true; + } + + // Otherwise it's a ref store. Load the previous value so we can show it + // to the post-write barrier. + // + // Optimisation opportunity: for the case where this field write results + // from struct.new, the old value is always zero. So we should synthesise + // a suitable zero constant rather than reading it from the object. See + // also bug 1799999. + MOZ_ASSERT(narrowingOp == MNarrowingOp::None); + MOZ_ASSERT(fieldType.widenToValType() == fieldType.valType()); + + // Store the new value + auto* store = MWasmStoreFieldRefKA::New( + alloc(), instancePointer_, keepAlive, base, offset, value, + AliasSet::Store(aliasBitset), mozilla::Some(getTrapSiteInfo()), + preBarrierKind); + if (!store) { + return false; + } + curBlock_->add(store); + + // Call the post-write barrier + return postBarrier(lineOrBytecode, keepAlive, base, offset, value); + } + + // Generate a write of `value` at address `base + index * scale`, where + // `scale` is known at JIT-time. If the written value is a reftype, the + // previous value at `base + index * scale` will be retrieved and handed off + // to the post-write barrier. `keepAlive` will be referenced by the + // instruction so as to hold it live (from the GC's point of view). + [[nodiscard]] bool writeGcValueAtBasePlusScaledIndex( + uint32_t lineOrBytecode, FieldType fieldType, MDefinition* keepAlive, + AliasSet::Flag aliasBitset, MDefinition* value, MDefinition* base, + uint32_t scale, MDefinition* index, WasmPreBarrierKind preBarrierKind) { + MOZ_ASSERT(aliasBitset != 0); + MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull); + MOZ_ASSERT(fieldType.widenToValType().toMIRType() == value->type()); + MOZ_ASSERT(scale == 1 || scale == 2 || scale == 4 || scale == 8 || + scale == 16); + + // Currently there's no single MIR node that this can be translated into. + // So compute the final address "manually", then store directly to that + // address. See bug 1802287. + MDefinition* scaleDef = constantTargetWord(intptr_t(scale)); + if (!scaleDef) { + return false; + } + MDefinition* finalAddr = computeBasePlusScaledIndex(base, scaleDef, index); + if (!finalAddr) { + return false; + } + + return writeGcValueAtBasePlusOffset( + lineOrBytecode, fieldType, keepAlive, aliasBitset, value, finalAddr, + /*offset=*/0, + /*needsTrapInfo=*/false, preBarrierKind); + } + + // Generate a read from address `base + offset`, where `offset` is known at + // JIT time. The loaded value will be widened as described by `fieldType` + // and `fieldWideningOp`. `keepAlive` will be referenced by the instruction + // so as to hold it live (from the GC's point of view). + [[nodiscard]] MDefinition* readGcValueAtBasePlusOffset( + FieldType fieldType, FieldWideningOp fieldWideningOp, + MDefinition* keepAlive, AliasSet::Flag aliasBitset, MDefinition* base, + uint32_t offset, bool needsTrapInfo) { + MOZ_ASSERT(aliasBitset != 0); + MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull); + MIRType mirType; + MWideningOp mirWideningOp; + fieldLoadInfoToMIR(fieldType, fieldWideningOp, &mirType, &mirWideningOp); + MaybeTrapSiteInfo maybeTrap; + if (needsTrapInfo) { + maybeTrap.emplace(getTrapSiteInfo()); + } + auto* load = MWasmLoadFieldKA::New(alloc(), keepAlive, base, offset, + mirType, mirWideningOp, + AliasSet::Load(aliasBitset), maybeTrap); + if (!load) { + return nullptr; + } + curBlock_->add(load); + return load; + } + + // Generate a read from address `base + index * scale`, where `scale` is + // known at JIT-time. The loaded value will be widened as described by + // `fieldType` and `fieldWideningOp`. `keepAlive` will be referenced by the + // instruction so as to hold it live (from the GC's point of view). + [[nodiscard]] MDefinition* readGcValueAtBasePlusScaledIndex( + FieldType fieldType, FieldWideningOp fieldWideningOp, + MDefinition* keepAlive, AliasSet::Flag aliasBitset, MDefinition* base, + uint32_t scale, MDefinition* index) { + MOZ_ASSERT(aliasBitset != 0); + MOZ_ASSERT(keepAlive->type() == MIRType::RefOrNull); + MOZ_ASSERT(scale == 1 || scale == 2 || scale == 4 || scale == 8 || + scale == 16); + + // Currently there's no single MIR node that this can be translated into. + // So compute the final address "manually", then store directly to that + // address. See bug 1802287. + MDefinition* scaleDef = constantTargetWord(intptr_t(scale)); + if (!scaleDef) { + return nullptr; + } + MDefinition* finalAddr = computeBasePlusScaledIndex(base, scaleDef, index); + if (!finalAddr) { + return nullptr; + } + + MIRType mirType; + MWideningOp mirWideningOp; + fieldLoadInfoToMIR(fieldType, fieldWideningOp, &mirType, &mirWideningOp); + auto* load = MWasmLoadFieldKA::New(alloc(), keepAlive, finalAddr, + /*offset=*/0, mirType, mirWideningOp, + AliasSet::Load(aliasBitset), + mozilla::Some(getTrapSiteInfo())); + if (!load) { + return nullptr; + } + curBlock_->add(load); + return load; + } + + /************************************************ WasmGC: type helpers ***/ + + // Returns an MDefinition holding the supertype vector for `typeIndex`. + [[nodiscard]] MDefinition* loadSuperTypeVector(uint32_t typeIndex) { + uint32_t superTypeVectorOffset = + moduleEnv().offsetOfSuperTypeVector(typeIndex); + + auto* load = MWasmLoadInstanceDataField::New( + alloc(), MIRType::Pointer, superTypeVectorOffset, + /*isConst=*/true, instancePointer_); + if (!load) { + return nullptr; + } + curBlock_->add(load); + return load; + } + + [[nodiscard]] MDefinition* loadTypeDefInstanceData(uint32_t typeIndex) { + size_t offset = Instance::offsetInData( + moduleEnv_.offsetOfTypeDefInstanceData(typeIndex)); + auto* result = MWasmDerivedPointer::New(alloc(), instancePointer_, offset); + if (!result) { + return nullptr; + } + curBlock_->add(result); + return result; + } + + /********************************************** WasmGC: struct helpers ***/ + + // Helper function for EmitStruct{New,Set}: given a MIR pointer to a + // WasmStructObject, a MIR pointer to a value, and a field descriptor, + // generate MIR to write the value to the relevant field in the object. + [[nodiscard]] bool writeValueToStructField( + uint32_t lineOrBytecode, const StructField& field, + MDefinition* structObject, MDefinition* value, + WasmPreBarrierKind preBarrierKind) { + FieldType fieldType = field.type; + uint32_t fieldOffset = field.offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + // Make `base` point at the first byte of either the struct object as a + // whole or of the out-of-line data area. And adjust `areaOffset` + // accordingly. + MDefinition* base; + bool needsTrapInfo; + if (areaIsOutline) { + auto* load = MWasmLoadField::New( + alloc(), structObject, WasmStructObject::offsetOfOutlineData(), + MIRType::Pointer, MWideningOp::None, + AliasSet::Load(AliasSet::WasmStructOutlineDataPointer), + mozilla::Some(getTrapSiteInfo())); + if (!load) { + return false; + } + curBlock_->add(load); + base = load; + needsTrapInfo = false; + } else { + base = structObject; + needsTrapInfo = true; + areaOffset += WasmStructObject::offsetOfInlineData(); + } + // The transaction is to happen at `base + areaOffset`, so to speak. + // After this point we must ignore `fieldOffset`. + + // The alias set denoting the field's location, although lacking a + // Load-vs-Store indication at this point. + AliasSet::Flag fieldAliasSet = areaIsOutline + ? AliasSet::WasmStructOutlineDataArea + : AliasSet::WasmStructInlineDataArea; + + return writeGcValueAtBasePlusOffset(lineOrBytecode, fieldType, structObject, + fieldAliasSet, value, base, areaOffset, + needsTrapInfo, preBarrierKind); + } + + // Helper function for EmitStructGet: given a MIR pointer to a + // WasmStructObject, a field descriptor and a field widening operation, + // generate MIR to read the value from the relevant field in the object. + [[nodiscard]] MDefinition* readValueFromStructField( + const StructField& field, FieldWideningOp wideningOp, + MDefinition* structObject) { + FieldType fieldType = field.type; + uint32_t fieldOffset = field.offset; + + bool areaIsOutline; + uint32_t areaOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(fieldType, fieldOffset, + &areaIsOutline, &areaOffset); + + // Make `base` point at the first byte of either the struct object as a + // whole or of the out-of-line data area. And adjust `areaOffset` + // accordingly. + MDefinition* base; + bool needsTrapInfo; + if (areaIsOutline) { + auto* loadOOLptr = MWasmLoadField::New( + alloc(), structObject, WasmStructObject::offsetOfOutlineData(), + MIRType::Pointer, MWideningOp::None, + AliasSet::Load(AliasSet::WasmStructOutlineDataPointer), + mozilla::Some(getTrapSiteInfo())); + if (!loadOOLptr) { + return nullptr; + } + curBlock_->add(loadOOLptr); + base = loadOOLptr; + needsTrapInfo = false; + } else { + base = structObject; + needsTrapInfo = true; + areaOffset += WasmStructObject::offsetOfInlineData(); + } + // The transaction is to happen at `base + areaOffset`, so to speak. + // After this point we must ignore `fieldOffset`. + + // The alias set denoting the field's location, although lacking a + // Load-vs-Store indication at this point. + AliasSet::Flag fieldAliasSet = areaIsOutline + ? AliasSet::WasmStructOutlineDataArea + : AliasSet::WasmStructInlineDataArea; + + return readGcValueAtBasePlusOffset(fieldType, wideningOp, structObject, + fieldAliasSet, base, areaOffset, + needsTrapInfo); + } + + /********************************* WasmGC: address-arithmetic helpers ***/ + + inline bool targetIs64Bit() const { +#ifdef JS_64BIT + return true; +#else + return false; +#endif + } + + // Generate MIR to unsigned widen `val` out to the target word size. If + // `val` is already at the target word size, this is a no-op. The only + // other allowed case is where `val` is Int32 and we're compiling for a + // 64-bit target, in which case a widen is generated. + [[nodiscard]] MDefinition* unsignedWidenToTargetWord(MDefinition* val) { + if (targetIs64Bit()) { + if (val->type() == MIRType::Int32) { + auto* ext = MExtendInt32ToInt64::New(alloc(), val, /*isUnsigned=*/true); + if (!ext) { + return nullptr; + } + curBlock_->add(ext); + return ext; + } + MOZ_ASSERT(val->type() == MIRType::Int64); + return val; + } + MOZ_ASSERT(val->type() == MIRType::Int32); + return val; + } + + // Compute `base + index * scale`, for both 32- and 64-bit targets. For the + // convenience of callers, on a 64-bit target, `index` and `scale` can + // (independently) be either Int32 or Int64; in the former case they will be + // zero-extended before the multiplication, so that both the multiplication + // and addition are done at the target word size. + [[nodiscard]] MDefinition* computeBasePlusScaledIndex(MDefinition* base, + MDefinition* scale, + MDefinition* index) { + // On a 32-bit target, require: + // base : Int32 (== TargetWordMIRType()) + // index, scale : Int32 + // Calculate base +32 (index *32 scale) + // + // On a 64-bit target, require: + // base : Int64 (== TargetWordMIRType()) + // index, scale: either Int32 or Int64 (any combination is OK) + // Calculate base +64 (u-widen to 64(index)) *64 (u-widen to 64(scale)) + // + // Final result type is the same as that of `base`. + + MOZ_ASSERT(base->type() == TargetWordMIRType()); + + // Widen `index` if necessary, producing `indexW`. + MDefinition* indexW = unsignedWidenToTargetWord(index); + if (!indexW) { + return nullptr; + } + // Widen `scale` if necessary, producing `scaleW`. + MDefinition* scaleW = unsignedWidenToTargetWord(scale); + if (!scaleW) { + return nullptr; + } + // Compute `scaledIndex = indexW * scaleW`. + MIRType targetWordType = TargetWordMIRType(); + bool targetIs64 = targetWordType == MIRType::Int64; + MMul* scaledIndex = + MMul::NewWasm(alloc(), indexW, scaleW, targetWordType, + targetIs64 ? MMul::Mode::Normal : MMul::Mode::Integer, + /*mustPreserveNan=*/false); + if (!scaledIndex) { + return nullptr; + } + // Compute `result = base + scaledIndex`. + curBlock_->add(scaledIndex); + MAdd* result = MAdd::NewWasm(alloc(), base, scaledIndex, targetWordType); + if (!result) { + return nullptr; + } + curBlock_->add(result); + return result; + } + + /********************************************** WasmGC: array helpers ***/ + + // Given `arrayObject`, the address of a WasmArrayObject, generate MIR to + // return the contents of the WasmArrayObject::numElements_ field. + // Adds trap site info for the null check. + [[nodiscard]] MDefinition* getWasmArrayObjectNumElements( + MDefinition* arrayObject) { + MOZ_ASSERT(arrayObject->type() == MIRType::RefOrNull); + + auto* numElements = MWasmLoadField::New( + alloc(), arrayObject, WasmArrayObject::offsetOfNumElements(), + MIRType::Int32, MWideningOp::None, + AliasSet::Load(AliasSet::WasmArrayNumElements), + mozilla::Some(getTrapSiteInfo())); + if (!numElements) { + return nullptr; + } + curBlock_->add(numElements); + + return numElements; + } + + // Given `arrayObject`, the address of a WasmArrayObject, generate MIR to + // return the contents of the WasmArrayObject::data_ field. + [[nodiscard]] MDefinition* getWasmArrayObjectData(MDefinition* arrayObject) { + MOZ_ASSERT(arrayObject->type() == MIRType::RefOrNull); + + auto* data = MWasmLoadField::New( + alloc(), arrayObject, WasmArrayObject::offsetOfData(), + TargetWordMIRType(), MWideningOp::None, + AliasSet::Load(AliasSet::WasmArrayDataPointer), + mozilla::Some(getTrapSiteInfo())); + if (!data) { + return nullptr; + } + curBlock_->add(data); + + return data; + } + + // Given a JIT-time-known type index `typeIndex` and a run-time known number + // of elements `numElements`, create MIR to call `Instance::arrayNew`, + // producing an array with the relevant type and size and initialized with + // `typeIndex`s default value. + [[nodiscard]] MDefinition* createDefaultInitializedArrayObject( + uint32_t lineOrBytecode, uint32_t typeIndex, MDefinition* numElements) { + // Get the type definition for the array as a whole. + MDefinition* typeDefData = loadTypeDefInstanceData(typeIndex); + if (!typeDefData) { + return nullptr; + } + + // Create call: arrayObject = Instance::arrayNew(numElements, typeDefData) + // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated + // by this call will trap. + MDefinition* arrayObject; + if (!emitInstanceCall2(lineOrBytecode, SASigArrayNew, numElements, + typeDefData, &arrayObject)) { + return nullptr; + } + + return arrayObject; + } + + [[nodiscard]] MDefinition* createUninitializedArrayObject( + uint32_t lineOrBytecode, uint32_t typeIndex, MDefinition* numElements) { + // Get the type definition for the array as a whole. + MDefinition* typeDefData = loadTypeDefInstanceData(typeIndex); + if (!typeDefData) { + return nullptr; + } + + // Create call: arrayObject = Instance::arrayNewUninit(numElements, + // typeDefData) If the requested size exceeds MaxArrayPayloadBytes, the MIR + // generated by this call will trap. + MDefinition* arrayObject; + if (!emitInstanceCall2(lineOrBytecode, SASigArrayNewUninit, numElements, + typeDefData, &arrayObject)) { + return nullptr; + } + + return arrayObject; + } + + // This emits MIR to perform several actions common to array loads and + // stores. Given `arrayObject`, that points to a WasmArrayObject, and an + // index value `index`, it: + // + // * Generates a trap if the array pointer is null + // * Gets the size of the array + // * Emits a bounds check of `index` against the array size + // * Retrieves the OOL object pointer from the array + // * Includes check for null via signal handler. + // + // The returned value is for the OOL object pointer. + [[nodiscard]] MDefinition* setupForArrayAccess(MDefinition* arrayObject, + MDefinition* index) { + MOZ_ASSERT(arrayObject->type() == MIRType::RefOrNull); + MOZ_ASSERT(index->type() == MIRType::Int32); + + // Check for null is done in getWasmArrayObjectNumElements. + + // Get the size value for the array. + MDefinition* numElements = getWasmArrayObjectNumElements(arrayObject); + if (!numElements) { + return nullptr; + } + + // Create a bounds check. + auto* boundsCheck = + MWasmBoundsCheck::New(alloc(), index, numElements, bytecodeOffset(), + MWasmBoundsCheck::Target::Unknown); + if (!boundsCheck) { + return nullptr; + } + curBlock_->add(boundsCheck); + + // Get the address of the first byte of the (OOL) data area. + return getWasmArrayObjectData(arrayObject); + } + + // This routine generates all MIR required for `array.new`. The returned + // value is for the newly created array. + [[nodiscard]] MDefinition* createArrayNewCallAndLoop(uint32_t lineOrBytecode, + uint32_t typeIndex, + MDefinition* numElements, + MDefinition* fillValue) { + const ArrayType& arrayType = (*moduleEnv_.types)[typeIndex].arrayType(); + + // Create the array object, default-initialized. + MDefinition* arrayObject = + createUninitializedArrayObject(lineOrBytecode, typeIndex, numElements); + if (!arrayObject) { + return nullptr; + } + + mozilla::DebugOnly fillValueMIRType = fillValue->type(); + FieldType fillValueFieldType = arrayType.elementType_; + MOZ_ASSERT(fillValueFieldType.widenToValType().toMIRType() == + fillValueMIRType); + + uint32_t elemSize = fillValueFieldType.size(); + MOZ_ASSERT(elemSize >= 1 && elemSize <= 16); + + // Make `base` point at the first byte of the (OOL) data area. + MDefinition* base = getWasmArrayObjectData(arrayObject); + if (!base) { + return nullptr; + } + + // We have: + // base : TargetWord + // numElements : Int32 + // fillValue : + // $elemSize = arrayType.elementType_.size(); 1, 2, 4, 8 or 16 + // + // Generate MIR: + // + // limit : TargetWord = base + nElems * elemSize + // if (limit == base) goto after; // skip loop if trip count == 0 + // // optimisation (not done): skip loop if fill value == 0 + // loop: + // ptrPhi = phi(base, ptrNext) + // *ptrPhi = fillValue + // ptrNext = ptrPhi + $elemSize + // if (ptrNext end(skipIfLimitEqualsBase); + if (!afterBlock->addPredecessor(alloc(), curBlock_)) { + return nullptr; + } + // Optimisation opportunity: if the fill value is zero, maybe we should + // likewise skip over the initialisation loop entirely (and, if the zero + // value is visible at JIT time, the loop will be removed). For the + // reftyped case, that would be a big win since each iteration requires a + // call to the post-write barrier routine. + + // ==== Fill in the loop block as best we can ==== + curBlock_ = loopBlock; + MPhi* ptrPhi = MPhi::New(alloc(), TargetWordMIRType()); + if (!ptrPhi) { + return nullptr; + } + if (!ptrPhi->reserveLength(2)) { + return nullptr; + } + ptrPhi->addInput(base); + curBlock_->addPhi(ptrPhi); + curBlock_->setLoopDepth(loopDepth_ + 1); + + // Because we have the exact address to hand, use + // `writeGcValueAtBasePlusOffset` rather than + // `writeGcValueAtBasePlusScaledIndex` to do the store. + if (!writeGcValueAtBasePlusOffset( + lineOrBytecode, fillValueFieldType, arrayObject, + AliasSet::WasmArrayDataArea, fillValue, ptrPhi, /*offset=*/0, + /*needsTrapInfo=*/false, WasmPreBarrierKind::None)) { + return nullptr; + } + + auto* ptrNext = + MAdd::NewWasm(alloc(), ptrPhi, elemSizeDef, TargetWordMIRType()); + if (!ptrNext) { + return nullptr; + } + curBlock_->add(ptrNext); + ptrPhi->addInput(ptrNext); + + MDefinition* ptrNextLtuLimit = compare( + ptrNext, limit, JSOp::Lt, + targetIs64Bit() ? MCompare::Compare_UInt64 : MCompare::Compare_UInt32); + if (!ptrNextLtuLimit) { + return nullptr; + } + auto* continueIfPtrNextLtuLimit = + MTest::New(alloc(), ptrNextLtuLimit, loopBlock, afterBlock); + if (!continueIfPtrNextLtuLimit) { + return nullptr; + } + curBlock_->end(continueIfPtrNextLtuLimit); + if (!loopBlock->addPredecessor(alloc(), loopBlock)) { + return nullptr; + } + // ==== Loop block completed ==== + + curBlock_ = afterBlock; + return arrayObject; + } + + /*********************************************** WasmGC: other helpers ***/ + + // Generate MIR that causes a trap of kind `trapKind` if `arg` is zero. + // Currently `arg` may only be a MIRType::Int32, but that requirement could + // be relaxed if needed in future. + [[nodiscard]] bool trapIfZero(wasm::Trap trapKind, MDefinition* arg) { + MOZ_ASSERT(arg->type() == MIRType::Int32); + + MBasicBlock* trapBlock = nullptr; + if (!newBlock(curBlock_, &trapBlock)) { + return false; + } + + auto* trap = MWasmTrap::New(alloc(), trapKind, bytecodeOffset()); + if (!trap) { + return false; + } + trapBlock->end(trap); + + MBasicBlock* joinBlock = nullptr; + if (!newBlock(curBlock_, &joinBlock)) { + return false; + } + + auto* test = MTest::New(alloc(), arg, joinBlock, trapBlock); + if (!test) { + return false; + } + curBlock_->end(test); + curBlock_ = joinBlock; + return true; + } + + [[nodiscard]] MDefinition* isGcObjectSubtypeOf(MDefinition* object, + RefType sourceType, + RefType destType) { + MInstruction* isSubTypeOf = nullptr; + if (destType.isTypeRef()) { + uint32_t typeIndex = moduleEnv_.types->indexOf(*destType.typeDef()); + MDefinition* superSuperTypeVector = loadSuperTypeVector(typeIndex); + isSubTypeOf = MWasmGcObjectIsSubtypeOfConcrete::New( + alloc(), object, superSuperTypeVector, sourceType, destType); + } else { + isSubTypeOf = MWasmGcObjectIsSubtypeOfAbstract::New(alloc(), object, + sourceType, destType); + } + MOZ_ASSERT(isSubTypeOf); + + curBlock_->add(isSubTypeOf); + return isSubTypeOf; + } + + // Generate MIR that attempts to downcast `ref` to `castToTypeDef`. If the + // downcast fails, we trap. If it succeeds, then `ref` can be assumed to + // have a type that is a subtype of (or the same as) `castToTypeDef` after + // this point. + [[nodiscard]] bool refCast(MDefinition* ref, RefType sourceType, + RefType destType) { + MDefinition* success = isGcObjectSubtypeOf(ref, sourceType, destType); + if (!success) { + return false; + } + + // Trap if `success` is zero. If it's nonzero, we have established that + // `ref <: castToTypeDef`. + return trapIfZero(wasm::Trap::BadCast, success); + } + + // Generate MIR that computes a boolean value indicating whether or not it + // is possible to downcast `ref` to `destType`. + [[nodiscard]] MDefinition* refTest(MDefinition* ref, RefType sourceType, + RefType destType) { + return isGcObjectSubtypeOf(ref, sourceType, destType); + } + + // Generates MIR for br_on_cast and br_on_cast_fail. + [[nodiscard]] bool brOnCastCommon(bool onSuccess, uint32_t labelRelativeDepth, + RefType sourceType, RefType destType, + const ResultType& labelType, + const DefVector& values) { + if (inDeadCode()) { + return true; + } + + MBasicBlock* fallthroughBlock = nullptr; + if (!newBlock(curBlock_, &fallthroughBlock)) { + return false; + } + + // `values` are the values in the top block-value on the stack. Since the + // argument to `br_on_cast{_fail}` is at the top of the stack, it is the + // last element in `values`. + // + // For both br_on_cast and br_on_cast_fail, the OpIter validation routines + // ensure that `values` is non-empty (by rejecting the case + // `labelType->length() < 1`) and that the last value in `values` is + // reftyped. + MOZ_RELEASE_ASSERT(values.length() > 0); + MDefinition* ref = values.back(); + MOZ_ASSERT(ref->type() == MIRType::RefOrNull); + + MDefinition* success = isGcObjectSubtypeOf(ref, sourceType, destType); + if (!success) { + return false; + } + + MTest* test; + if (onSuccess) { + test = MTest::New(alloc(), success, nullptr, fallthroughBlock); + if (!test || !addControlFlowPatch(test, labelRelativeDepth, + MTest::TrueBranchIndex)) { + return false; + } + } else { + test = MTest::New(alloc(), success, fallthroughBlock, nullptr); + if (!test || !addControlFlowPatch(test, labelRelativeDepth, + MTest::FalseBranchIndex)) { + return false; + } + } + + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(test); + curBlock_ = fallthroughBlock; + return true; + } + + [[nodiscard]] bool brOnNonStruct(const DefVector& values) { + if (inDeadCode()) { + return true; + } + + MBasicBlock* fallthroughBlock = nullptr; + if (!newBlock(curBlock_, &fallthroughBlock)) { + return false; + } + + MOZ_ASSERT(values.length() > 0); + MOZ_ASSERT(values.back()->type() == MIRType::RefOrNull); + + MGoto* jump = MGoto::New(alloc(), fallthroughBlock); + if (!jump) { + return false; + } + if (!pushDefs(values)) { + return false; + } + + curBlock_->end(jump); + curBlock_ = fallthroughBlock; + return true; + } + + /************************************************************ DECODING ***/ + + // AsmJS adds a line number to `callSiteLineNums` for certain operations that + // are represented by a JS call, such as math builtins. We use these line + // numbers when calling builtins. This method will read from + // `callSiteLineNums` when we are using AsmJS, or else return the current + // bytecode offset. + // + // This method MUST be called from opcodes that AsmJS will emit a call site + // line number for, or else the arrays will get out of sync. Other opcodes + // must use `readBytecodeOffset` below. + uint32_t readCallSiteLineOrBytecode() { + if (!func_.callSiteLineNums.empty()) { + return func_.callSiteLineNums[lastReadCallSite_++]; + } + return iter_.lastOpcodeOffset(); + } + + // Return the current bytecode offset. + uint32_t readBytecodeOffset() { return iter_.lastOpcodeOffset(); } + + TrapSiteInfo getTrapSiteInfo() { + return TrapSiteInfo(wasm::BytecodeOffset(readBytecodeOffset())); + } + +#if DEBUG + bool done() const { return iter_.done(); } +#endif + + /*************************************************************************/ + private: + [[nodiscard]] bool newBlock(MBasicBlock* pred, MBasicBlock** block, + MBasicBlock::Kind kind = MBasicBlock::NORMAL) { + *block = MBasicBlock::New(mirGraph(), info(), pred, kind); + if (!*block) { + return false; + } + mirGraph().addBlock(*block); + (*block)->setLoopDepth(loopDepth_); + return true; + } + + [[nodiscard]] bool goToNewBlock(MBasicBlock* pred, MBasicBlock** block) { + if (!newBlock(pred, block)) { + return false; + } + pred->end(MGoto::New(alloc(), *block)); + return true; + } + + [[nodiscard]] bool goToExistingBlock(MBasicBlock* prev, MBasicBlock* next) { + MOZ_ASSERT(prev); + MOZ_ASSERT(next); + prev->end(MGoto::New(alloc(), next)); + return next->addPredecessor(alloc(), prev); + } + + [[nodiscard]] bool bindBranches(uint32_t absolute, DefVector* defs) { + if (absolute >= blockPatches_.length() || blockPatches_[absolute].empty()) { + return inDeadCode() || popPushedDefs(defs); + } + + ControlFlowPatchVector& patches = blockPatches_[absolute]; + MControlInstruction* ins = patches[0].ins; + MBasicBlock* pred = ins->block(); + + MBasicBlock* join = nullptr; + if (!newBlock(pred, &join)) { + return false; + } + + pred->mark(); + ins->replaceSuccessor(patches[0].index, join); + + for (size_t i = 1; i < patches.length(); i++) { + ins = patches[i].ins; + + pred = ins->block(); + if (!pred->isMarked()) { + if (!join->addPredecessor(alloc(), pred)) { + return false; + } + pred->mark(); + } + + ins->replaceSuccessor(patches[i].index, join); + } + + MOZ_ASSERT_IF(curBlock_, !curBlock_->isMarked()); + for (uint32_t i = 0; i < join->numPredecessors(); i++) { + join->getPredecessor(i)->unmark(); + } + + if (curBlock_ && !goToExistingBlock(curBlock_, join)) { + return false; + } + + curBlock_ = join; + + if (!popPushedDefs(defs)) { + return false; + } + + patches.clear(); + return true; + } +}; + +template <> +MDefinition* FunctionCompiler::unary(MDefinition* op) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MToFloat32::New(alloc(), op, mustPreserveNaN(op->type())); + curBlock_->add(ins); + return ins; +} + +template <> +MDefinition* FunctionCompiler::unary( + MDefinition* op) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MWasmBuiltinTruncateToInt32::New(alloc(), op, instancePointer_, + bytecodeOffset()); + curBlock_->add(ins); + return ins; +} + +template <> +MDefinition* FunctionCompiler::unary(MDefinition* op) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MNot::NewInt32(alloc(), op); + curBlock_->add(ins); + return ins; +} + +template <> +MDefinition* FunctionCompiler::unary(MDefinition* op, MIRType type) { + if (inDeadCode()) { + return nullptr; + } + auto* ins = MAbs::NewWasm(alloc(), op, type); + curBlock_->add(ins); + return ins; +} + +} // end anonymous namespace + +static bool EmitI32Const(FunctionCompiler& f) { + int32_t i32; + if (!f.iter().readI32Const(&i32)) { + return false; + } + + f.iter().setResult(f.constantI32(i32)); + return true; +} + +static bool EmitI64Const(FunctionCompiler& f) { + int64_t i64; + if (!f.iter().readI64Const(&i64)) { + return false; + } + + f.iter().setResult(f.constantI64(i64)); + return true; +} + +static bool EmitF32Const(FunctionCompiler& f) { + float f32; + if (!f.iter().readF32Const(&f32)) { + return false; + } + + f.iter().setResult(f.constantF32(f32)); + return true; +} + +static bool EmitF64Const(FunctionCompiler& f) { + double f64; + if (!f.iter().readF64Const(&f64)) { + return false; + } + + f.iter().setResult(f.constantF64(f64)); + return true; +} + +static bool EmitBlock(FunctionCompiler& f) { + ResultType params; + return f.iter().readBlock(¶ms) && f.startBlock(); +} + +static bool EmitLoop(FunctionCompiler& f) { + ResultType params; + if (!f.iter().readLoop(¶ms)) { + return false; + } + + MBasicBlock* loopHeader; + if (!f.startLoop(&loopHeader, params.length())) { + return false; + } + + f.addInterruptCheck(); + + f.iter().controlItem().setBlock(loopHeader); + return true; +} + +static bool EmitIf(FunctionCompiler& f) { + ResultType params; + MDefinition* condition = nullptr; + if (!f.iter().readIf(¶ms, &condition)) { + return false; + } + + MBasicBlock* elseBlock; + if (!f.branchAndStartThen(condition, &elseBlock)) { + return false; + } + + f.iter().controlItem().setBlock(elseBlock); + return true; +} + +static bool EmitElse(FunctionCompiler& f) { + ResultType paramType; + ResultType resultType; + DefVector thenValues; + if (!f.iter().readElse(¶mType, &resultType, &thenValues)) { + return false; + } + + if (!f.pushDefs(thenValues)) { + return false; + } + + Control& control = f.iter().controlItem(); + if (!f.switchToElse(control.block, &control.block)) { + return false; + } + + return true; +} + +static bool EmitEnd(FunctionCompiler& f) { + LabelKind kind; + ResultType type; + DefVector preJoinDefs; + DefVector resultsForEmptyElse; + if (!f.iter().readEnd(&kind, &type, &preJoinDefs, &resultsForEmptyElse)) { + return false; + } + + Control& control = f.iter().controlItem(); + MBasicBlock* block = control.block; + + if (!f.pushDefs(preJoinDefs)) { + return false; + } + + // Every label case is responsible to pop the control item at the appropriate + // time for the label case + DefVector postJoinDefs; + switch (kind) { + case LabelKind::Body: + if (!f.emitBodyDelegateThrowPad(control)) { + return false; + } + if (!f.finishBlock(&postJoinDefs)) { + return false; + } + if (!f.returnValues(postJoinDefs)) { + return false; + } + f.iter().popEnd(); + MOZ_ASSERT(f.iter().controlStackEmpty()); + return f.iter().endFunction(f.iter().end()); + case LabelKind::Block: + if (!f.finishBlock(&postJoinDefs)) { + return false; + } + f.iter().popEnd(); + break; + case LabelKind::Loop: + if (!f.closeLoop(block, &postJoinDefs)) { + return false; + } + f.iter().popEnd(); + break; + case LabelKind::Then: { + // If we didn't see an Else, create a trivial else block so that we create + // a diamond anyway, to preserve Ion invariants. + if (!f.switchToElse(block, &block)) { + return false; + } + + if (!f.pushDefs(resultsForEmptyElse)) { + return false; + } + + if (!f.joinIfElse(block, &postJoinDefs)) { + return false; + } + f.iter().popEnd(); + break; + } + case LabelKind::Else: + if (!f.joinIfElse(block, &postJoinDefs)) { + return false; + } + f.iter().popEnd(); + break; + case LabelKind::Try: + case LabelKind::Catch: + case LabelKind::CatchAll: + if (!f.finishTryCatch(kind, control, &postJoinDefs)) { + return false; + } + f.iter().popEnd(); + break; + } + + MOZ_ASSERT_IF(!f.inDeadCode(), postJoinDefs.length() == type.length()); + f.iter().setResults(postJoinDefs.length(), postJoinDefs); + + return true; +} + +static bool EmitBr(FunctionCompiler& f) { + uint32_t relativeDepth; + ResultType type; + DefVector values; + if (!f.iter().readBr(&relativeDepth, &type, &values)) { + return false; + } + + return f.br(relativeDepth, values); +} + +static bool EmitBrIf(FunctionCompiler& f) { + uint32_t relativeDepth; + ResultType type; + DefVector values; + MDefinition* condition; + if (!f.iter().readBrIf(&relativeDepth, &type, &values, &condition)) { + return false; + } + + return f.brIf(relativeDepth, values, condition); +} + +static bool EmitBrTable(FunctionCompiler& f) { + Uint32Vector depths; + uint32_t defaultDepth; + ResultType branchValueType; + DefVector branchValues; + MDefinition* index; + if (!f.iter().readBrTable(&depths, &defaultDepth, &branchValueType, + &branchValues, &index)) { + return false; + } + + // If all the targets are the same, or there are no targets, we can just + // use a goto. This is not just an optimization: MaybeFoldConditionBlock + // assumes that tables have more than one successor. + bool allSameDepth = true; + for (uint32_t depth : depths) { + if (depth != defaultDepth) { + allSameDepth = false; + break; + } + } + + if (allSameDepth) { + return f.br(defaultDepth, branchValues); + } + + return f.brTable(index, defaultDepth, depths, branchValues); +} + +static bool EmitReturn(FunctionCompiler& f) { + DefVector values; + if (!f.iter().readReturn(&values)) { + return false; + } + + return f.returnValues(values); +} + +static bool EmitUnreachable(FunctionCompiler& f) { + if (!f.iter().readUnreachable()) { + return false; + } + + f.unreachableTrap(); + return true; +} + +static bool EmitTry(FunctionCompiler& f) { + ResultType params; + if (!f.iter().readTry(¶ms)) { + return false; + } + + MBasicBlock* curBlock = nullptr; + if (!f.startTry(&curBlock)) { + return false; + } + + f.iter().controlItem().setBlock(curBlock); + return true; +} + +static bool EmitCatch(FunctionCompiler& f) { + LabelKind kind; + uint32_t tagIndex; + ResultType paramType, resultType; + DefVector tryValues; + if (!f.iter().readCatch(&kind, &tagIndex, ¶mType, &resultType, + &tryValues)) { + return false; + } + + // Pushing the results of the previous block, to properly join control flow + // after the try and after each handler, as well as potential control flow + // patches from other instrunctions. This is similar to what is done for + // if-then-else control flow and for most other control control flow joins. + if (!f.pushDefs(tryValues)) { + return false; + } + + return f.switchToCatch(f.iter().controlItem(), kind, tagIndex); +} + +static bool EmitCatchAll(FunctionCompiler& f) { + LabelKind kind; + ResultType paramType, resultType; + DefVector tryValues; + if (!f.iter().readCatchAll(&kind, ¶mType, &resultType, &tryValues)) { + return false; + } + + // Pushing the results of the previous block, to properly join control flow + // after the try and after each handler, as well as potential control flow + // patches from other instrunctions. + if (!f.pushDefs(tryValues)) { + return false; + } + + return f.switchToCatch(f.iter().controlItem(), kind, CatchAllIndex); +} + +static bool EmitDelegate(FunctionCompiler& f) { + uint32_t relativeDepth; + ResultType resultType; + DefVector tryValues; + if (!f.iter().readDelegate(&relativeDepth, &resultType, &tryValues)) { + return false; + } + + Control& control = f.iter().controlItem(); + MBasicBlock* block = control.block; + + // Unless the entire try-delegate is dead code, delegate any pad-patches from + // this try to the next try-block above relativeDepth. + if (block) { + ControlInstructionVector& delegatePadPatches = control.tryPadPatches; + if (!f.delegatePadPatches(delegatePadPatches, relativeDepth)) { + return false; + } + } + f.iter().popDelegate(); + + // Push the results of the previous block, and join control flow with + // potential control flow patches from other instrunctions in the try code. + // This is similar to what is done for EmitEnd. + if (!f.pushDefs(tryValues)) { + return false; + } + DefVector postJoinDefs; + if (!f.finishBlock(&postJoinDefs)) { + return false; + } + MOZ_ASSERT_IF(!f.inDeadCode(), postJoinDefs.length() == resultType.length()); + f.iter().setResults(postJoinDefs.length(), postJoinDefs); + + return true; +} + +static bool EmitThrow(FunctionCompiler& f) { + uint32_t tagIndex; + DefVector argValues; + if (!f.iter().readThrow(&tagIndex, &argValues)) { + return false; + } + + return f.emitThrow(tagIndex, argValues); +} + +static bool EmitRethrow(FunctionCompiler& f) { + uint32_t relativeDepth; + if (!f.iter().readRethrow(&relativeDepth)) { + return false; + } + + return f.emitRethrow(relativeDepth); +} + +static bool EmitCallArgs(FunctionCompiler& f, const FuncType& funcType, + const DefVector& args, CallCompileState* call) { + for (size_t i = 0, n = funcType.args().length(); i < n; ++i) { + if (!f.mirGen().ensureBallast()) { + return false; + } + if (!f.passArg(args[i], funcType.args()[i], call)) { + return false; + } + } + + ResultType resultType = ResultType::Vector(funcType.results()); + if (!f.passStackResultAreaCallArg(resultType, call)) { + return false; + } + + return f.finishCall(call); +} + +static bool EmitCall(FunctionCompiler& f, bool asmJSFuncDef) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t funcIndex; + DefVector args; + if (asmJSFuncDef) { + if (!f.iter().readOldCallDirect(f.moduleEnv().numFuncImports, &funcIndex, + &args)) { + return false; + } + } else { + if (!f.iter().readCall(&funcIndex, &args)) { + return false; + } + } + + if (f.inDeadCode()) { + return true; + } + + const FuncType& funcType = *f.moduleEnv().funcs[funcIndex].type; + + CallCompileState call; + if (!EmitCallArgs(f, funcType, args, &call)) { + return false; + } + + DefVector results; + if (f.moduleEnv().funcIsImport(funcIndex)) { + uint32_t instanceDataOffset = + f.moduleEnv().offsetOfFuncImportInstanceData(funcIndex); + if (!f.callImport(instanceDataOffset, lineOrBytecode, call, funcType, + &results)) { + return false; + } + } else { + if (!f.callDirect(funcType, funcIndex, lineOrBytecode, call, &results)) { + return false; + } + } + + f.iter().setResults(results.length(), results); + return true; +} + +static bool EmitCallIndirect(FunctionCompiler& f, bool oldStyle) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t funcTypeIndex; + uint32_t tableIndex; + MDefinition* callee; + DefVector args; + if (oldStyle) { + tableIndex = 0; + if (!f.iter().readOldCallIndirect(&funcTypeIndex, &callee, &args)) { + return false; + } + } else { + if (!f.iter().readCallIndirect(&funcTypeIndex, &tableIndex, &callee, + &args)) { + return false; + } + } + + if (f.inDeadCode()) { + return true; + } + + const FuncType& funcType = (*f.moduleEnv().types)[funcTypeIndex].funcType(); + + CallCompileState call; + if (!EmitCallArgs(f, funcType, args, &call)) { + return false; + } + + DefVector results; + if (!f.callIndirect(funcTypeIndex, tableIndex, callee, lineOrBytecode, call, + &results)) { + return false; + } + + f.iter().setResults(results.length(), results); + return true; +} + +static bool EmitGetLocal(FunctionCompiler& f) { + uint32_t id; + if (!f.iter().readGetLocal(f.locals(), &id)) { + return false; + } + + f.iter().setResult(f.getLocalDef(id)); + return true; +} + +static bool EmitSetLocal(FunctionCompiler& f) { + uint32_t id; + MDefinition* value; + if (!f.iter().readSetLocal(f.locals(), &id, &value)) { + return false; + } + + f.assign(id, value); + return true; +} + +static bool EmitTeeLocal(FunctionCompiler& f) { + uint32_t id; + MDefinition* value; + if (!f.iter().readTeeLocal(f.locals(), &id, &value)) { + return false; + } + + f.assign(id, value); + return true; +} + +static bool EmitGetGlobal(FunctionCompiler& f) { + uint32_t id; + if (!f.iter().readGetGlobal(&id)) { + return false; + } + + const GlobalDesc& global = f.moduleEnv().globals[id]; + if (!global.isConstant()) { + f.iter().setResult(f.loadGlobalVar(global.offset(), !global.isMutable(), + global.isIndirect(), + global.type().toMIRType())); + return true; + } + + LitVal value = global.constantValue(); + + MDefinition* result; + switch (value.type().kind()) { + case ValType::I32: + result = f.constantI32(int32_t(value.i32())); + break; + case ValType::I64: + result = f.constantI64(int64_t(value.i64())); + break; + case ValType::F32: + result = f.constantF32(value.f32()); + break; + case ValType::F64: + result = f.constantF64(value.f64()); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + result = f.constantV128(value.v128()); + break; +#else + return f.iter().fail("Ion has no SIMD support yet"); +#endif + case ValType::Ref: + MOZ_ASSERT(value.ref().isNull()); + result = f.constantNullRef(); + break; + default: + MOZ_CRASH("unexpected type in EmitGetGlobal"); + } + + f.iter().setResult(result); + return true; +} + +static bool EmitSetGlobal(FunctionCompiler& f) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + uint32_t id; + MDefinition* value; + if (!f.iter().readSetGlobal(&id, &value)) { + return false; + } + + const GlobalDesc& global = f.moduleEnv().globals[id]; + MOZ_ASSERT(global.isMutable()); + return f.storeGlobalVar(bytecodeOffset, global.offset(), global.isIndirect(), + value); +} + +static bool EmitTeeGlobal(FunctionCompiler& f) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + uint32_t id; + MDefinition* value; + if (!f.iter().readTeeGlobal(&id, &value)) { + return false; + } + + const GlobalDesc& global = f.moduleEnv().globals[id]; + MOZ_ASSERT(global.isMutable()); + + return f.storeGlobalVar(bytecodeOffset, global.offset(), global.isIndirect(), + value); +} + +template +static bool EmitUnary(FunctionCompiler& f, ValType operandType) { + MDefinition* input; + if (!f.iter().readUnary(operandType, &input)) { + return false; + } + + f.iter().setResult(f.unary(input)); + return true; +} + +template +static bool EmitConversion(FunctionCompiler& f, ValType operandType, + ValType resultType) { + MDefinition* input; + if (!f.iter().readConversion(operandType, resultType, &input)) { + return false; + } + + f.iter().setResult(f.unary(input)); + return true; +} + +template +static bool EmitUnaryWithType(FunctionCompiler& f, ValType operandType, + MIRType mirType) { + MDefinition* input; + if (!f.iter().readUnary(operandType, &input)) { + return false; + } + + f.iter().setResult(f.unary(input, mirType)); + return true; +} + +template +static bool EmitConversionWithType(FunctionCompiler& f, ValType operandType, + ValType resultType, MIRType mirType) { + MDefinition* input; + if (!f.iter().readConversion(operandType, resultType, &input)) { + return false; + } + + f.iter().setResult(f.unary(input, mirType)); + return true; +} + +static bool EmitTruncate(FunctionCompiler& f, ValType operandType, + ValType resultType, bool isUnsigned, + bool isSaturating) { + MDefinition* input = nullptr; + if (!f.iter().readConversion(operandType, resultType, &input)) { + return false; + } + + TruncFlags flags = 0; + if (isUnsigned) { + flags |= TRUNC_UNSIGNED; + } + if (isSaturating) { + flags |= TRUNC_SATURATING; + } + if (resultType == ValType::I32) { + if (f.moduleEnv().isAsmJS()) { + if (input && (input->type() == MIRType::Double || + input->type() == MIRType::Float32)) { + f.iter().setResult(f.unary(input)); + } else { + f.iter().setResult(f.unary(input)); + } + } else { + f.iter().setResult(f.truncate(input, flags)); + } + } else { + MOZ_ASSERT(resultType == ValType::I64); + MOZ_ASSERT(!f.moduleEnv().isAsmJS()); +#if defined(JS_CODEGEN_ARM) + f.iter().setResult(f.truncateWithInstance(input, flags)); +#else + f.iter().setResult(f.truncate(input, flags)); +#endif + } + return true; +} + +static bool EmitSignExtend(FunctionCompiler& f, uint32_t srcSize, + uint32_t targetSize) { + MDefinition* input; + ValType type = targetSize == 4 ? ValType::I32 : ValType::I64; + if (!f.iter().readConversion(type, type, &input)) { + return false; + } + + f.iter().setResult(f.signExtend(input, srcSize, targetSize)); + return true; +} + +static bool EmitExtendI32(FunctionCompiler& f, bool isUnsigned) { + MDefinition* input; + if (!f.iter().readConversion(ValType::I32, ValType::I64, &input)) { + return false; + } + + f.iter().setResult(f.extendI32(input, isUnsigned)); + return true; +} + +static bool EmitConvertI64ToFloatingPoint(FunctionCompiler& f, + ValType resultType, MIRType mirType, + bool isUnsigned) { + MDefinition* input; + if (!f.iter().readConversion(ValType::I64, resultType, &input)) { + return false; + } + + f.iter().setResult(f.convertI64ToFloatingPoint(input, mirType, isUnsigned)); + return true; +} + +static bool EmitReinterpret(FunctionCompiler& f, ValType resultType, + ValType operandType, MIRType mirType) { + MDefinition* input; + if (!f.iter().readConversion(operandType, resultType, &input)) { + return false; + } + + f.iter().setResult(f.unary(input, mirType)); + return true; +} + +static bool EmitAdd(FunctionCompiler& f, ValType type, MIRType mirType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(type, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.add(lhs, rhs, mirType)); + return true; +} + +static bool EmitSub(FunctionCompiler& f, ValType type, MIRType mirType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(type, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.sub(lhs, rhs, mirType)); + return true; +} + +static bool EmitRotate(FunctionCompiler& f, ValType type, bool isLeftRotation) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(type, &lhs, &rhs)) { + return false; + } + + MDefinition* result = f.rotate(lhs, rhs, type.toMIRType(), isLeftRotation); + f.iter().setResult(result); + return true; +} + +static bool EmitBitNot(FunctionCompiler& f, ValType operandType) { + MDefinition* input; + if (!f.iter().readUnary(operandType, &input)) { + return false; + } + + f.iter().setResult(f.bitnot(input)); + return true; +} + +static bool EmitBitwiseAndOrXor(FunctionCompiler& f, ValType operandType, + MIRType mirType, + MWasmBinaryBitwise::SubOpcode subOpc) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.binary(lhs, rhs, mirType, subOpc)); + return true; +} + +template +static bool EmitShift(FunctionCompiler& f, ValType operandType, + MIRType mirType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.binary(lhs, rhs, mirType)); + return true; +} + +static bool EmitUrsh(FunctionCompiler& f, ValType operandType, + MIRType mirType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.ursh(lhs, rhs, mirType)); + return true; +} + +static bool EmitMul(FunctionCompiler& f, ValType operandType, MIRType mirType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult( + f.mul(lhs, rhs, mirType, + mirType == MIRType::Int32 ? MMul::Integer : MMul::Normal)); + return true; +} + +static bool EmitDiv(FunctionCompiler& f, ValType operandType, MIRType mirType, + bool isUnsigned) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.div(lhs, rhs, mirType, isUnsigned)); + return true; +} + +static bool EmitRem(FunctionCompiler& f, ValType operandType, MIRType mirType, + bool isUnsigned) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.mod(lhs, rhs, mirType, isUnsigned)); + return true; +} + +static bool EmitMinMax(FunctionCompiler& f, ValType operandType, + MIRType mirType, bool isMax) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.minMax(lhs, rhs, mirType, isMax)); + return true; +} + +static bool EmitCopySign(FunctionCompiler& f, ValType operandType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.binary(lhs, rhs, operandType.toMIRType())); + return true; +} + +static bool EmitComparison(FunctionCompiler& f, ValType operandType, + JSOp compareOp, MCompare::CompareType compareType) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readComparison(operandType, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.compare(lhs, rhs, compareOp, compareType)); + return true; +} + +static bool EmitSelect(FunctionCompiler& f, bool typed) { + StackType type; + MDefinition* trueValue; + MDefinition* falseValue; + MDefinition* condition; + if (!f.iter().readSelect(typed, &type, &trueValue, &falseValue, &condition)) { + return false; + } + + f.iter().setResult(f.select(trueValue, falseValue, condition)); + return true; +} + +static bool EmitLoad(FunctionCompiler& f, ValType type, Scalar::Type viewType) { + LinearMemoryAddress addr; + if (!f.iter().readLoad(type, Scalar::byteSize(viewType), &addr)) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, + f.bytecodeIfNotAsmJS()); + auto* ins = f.load(addr.base, &access, type); + if (!f.inDeadCode() && !ins) { + return false; + } + + f.iter().setResult(ins); + return true; +} + +static bool EmitStore(FunctionCompiler& f, ValType resultType, + Scalar::Type viewType) { + LinearMemoryAddress addr; + MDefinition* value; + if (!f.iter().readStore(resultType, Scalar::byteSize(viewType), &addr, + &value)) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, + f.bytecodeIfNotAsmJS()); + + f.store(addr.base, &access, value); + return true; +} + +static bool EmitTeeStore(FunctionCompiler& f, ValType resultType, + Scalar::Type viewType) { + LinearMemoryAddress addr; + MDefinition* value; + if (!f.iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr, + &value)) { + return false; + } + + MOZ_ASSERT(f.isMem32()); // asm.js opcode + MemoryAccessDesc access(viewType, addr.align, addr.offset, + f.bytecodeIfNotAsmJS()); + + f.store(addr.base, &access, value); + return true; +} + +static bool EmitTeeStoreWithCoercion(FunctionCompiler& f, ValType resultType, + Scalar::Type viewType) { + LinearMemoryAddress addr; + MDefinition* value; + if (!f.iter().readTeeStore(resultType, Scalar::byteSize(viewType), &addr, + &value)) { + return false; + } + + if (resultType == ValType::F32 && viewType == Scalar::Float64) { + value = f.unary(value); + } else if (resultType == ValType::F64 && viewType == Scalar::Float32) { + value = f.unary(value); + } else { + MOZ_CRASH("unexpected coerced store"); + } + + MOZ_ASSERT(f.isMem32()); // asm.js opcode + MemoryAccessDesc access(viewType, addr.align, addr.offset, + f.bytecodeIfNotAsmJS()); + + f.store(addr.base, &access, value); + return true; +} + +static bool TryInlineUnaryBuiltin(FunctionCompiler& f, SymbolicAddress callee, + MDefinition* input) { + if (!input) { + return false; + } + + MOZ_ASSERT(IsFloatingPointType(input->type())); + + RoundingMode mode; + if (!IsRoundingFunction(callee, &mode)) { + return false; + } + + if (!MNearbyInt::HasAssemblerSupport(mode)) { + return false; + } + + f.iter().setResult(f.nearbyInt(input, mode)); + return true; +} + +static bool EmitUnaryMathBuiltinCall(FunctionCompiler& f, + const SymbolicAddressSignature& callee) { + MOZ_ASSERT(callee.numArgs == 1); + + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + MDefinition* input; + if (!f.iter().readUnary(ValType::fromMIRType(callee.argTypes[0]), &input)) { + return false; + } + + if (TryInlineUnaryBuiltin(f, callee.identity, input)) { + return true; + } + + CallCompileState call; + if (!f.passArg(input, callee.argTypes[0], &call)) { + return false; + } + + if (!f.finishCall(&call)) { + return false; + } + + MDefinition* def; + if (!f.builtinCall(callee, lineOrBytecode, call, &def)) { + return false; + } + + f.iter().setResult(def); + return true; +} + +static bool EmitBinaryMathBuiltinCall(FunctionCompiler& f, + const SymbolicAddressSignature& callee) { + MOZ_ASSERT(callee.numArgs == 2); + MOZ_ASSERT(callee.argTypes[0] == callee.argTypes[1]); + + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + CallCompileState call; + MDefinition* lhs; + MDefinition* rhs; + // This call to readBinary assumes both operands have the same type. + if (!f.iter().readBinary(ValType::fromMIRType(callee.argTypes[0]), &lhs, + &rhs)) { + return false; + } + + if (!f.passArg(lhs, callee.argTypes[0], &call)) { + return false; + } + + if (!f.passArg(rhs, callee.argTypes[1], &call)) { + return false; + } + + if (!f.finishCall(&call)) { + return false; + } + + MDefinition* def; + if (!f.builtinCall(callee, lineOrBytecode, call, &def)) { + return false; + } + + f.iter().setResult(def); + return true; +} + +static bool EmitMemoryGrow(FunctionCompiler& f) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* delta; + if (!f.iter().readMemoryGrow(&delta)) { + return false; + } + + const SymbolicAddressSignature& callee = + f.isNoMemOrMem32() ? SASigMemoryGrowM32 : SASigMemoryGrowM64; + + MDefinition* ret; + if (!f.emitInstanceCall1(bytecodeOffset, callee, delta, &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitMemorySize(FunctionCompiler& f) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + if (!f.iter().readMemorySize()) { + return false; + } + + const SymbolicAddressSignature& callee = + f.isNoMemOrMem32() ? SASigMemorySizeM32 : SASigMemorySizeM64; + + MDefinition* ret; + if (!f.emitInstanceCall0(bytecodeOffset, callee, &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitAtomicCmpXchg(FunctionCompiler& f, ValType type, + Scalar::Type viewType) { + LinearMemoryAddress addr; + MDefinition* oldValue; + MDefinition* newValue; + if (!f.iter().readAtomicCmpXchg(&addr, type, byteSize(viewType), &oldValue, + &newValue)) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(), + Synchronization::Full()); + auto* ins = + f.atomicCompareExchangeHeap(addr.base, &access, type, oldValue, newValue); + if (!f.inDeadCode() && !ins) { + return false; + } + + f.iter().setResult(ins); + return true; +} + +static bool EmitAtomicLoad(FunctionCompiler& f, ValType type, + Scalar::Type viewType) { + LinearMemoryAddress addr; + if (!f.iter().readAtomicLoad(&addr, type, byteSize(viewType))) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(), + Synchronization::Load()); + auto* ins = f.load(addr.base, &access, type); + if (!f.inDeadCode() && !ins) { + return false; + } + + f.iter().setResult(ins); + return true; +} + +static bool EmitAtomicRMW(FunctionCompiler& f, ValType type, + Scalar::Type viewType, jit::AtomicOp op) { + LinearMemoryAddress addr; + MDefinition* value; + if (!f.iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(), + Synchronization::Full()); + auto* ins = f.atomicBinopHeap(op, addr.base, &access, type, value); + if (!f.inDeadCode() && !ins) { + return false; + } + + f.iter().setResult(ins); + return true; +} + +static bool EmitAtomicStore(FunctionCompiler& f, ValType type, + Scalar::Type viewType) { + LinearMemoryAddress addr; + MDefinition* value; + if (!f.iter().readAtomicStore(&addr, type, byteSize(viewType), &value)) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(), + Synchronization::Store()); + f.store(addr.base, &access, value); + return true; +} + +static bool EmitWait(FunctionCompiler& f, ValType type, uint32_t byteSize) { + MOZ_ASSERT(type == ValType::I32 || type == ValType::I64); + MOZ_ASSERT(type.size() == byteSize); + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + LinearMemoryAddress addr; + MDefinition* expected; + MDefinition* timeout; + if (!f.iter().readWait(&addr, type, byteSize, &expected, &timeout)) { + return false; + } + + MemoryAccessDesc access(type == ValType::I32 ? Scalar::Int32 : Scalar::Int64, + addr.align, addr.offset, f.bytecodeOffset()); + MDefinition* ptr = f.computeEffectiveAddress(addr.base, &access); + if (!f.inDeadCode() && !ptr) { + return false; + } + + const SymbolicAddressSignature& callee = + f.isNoMemOrMem32() + ? (type == ValType::I32 ? SASigWaitI32M32 : SASigWaitI64M32) + : (type == ValType::I32 ? SASigWaitI32M64 : SASigWaitI64M64); + MOZ_ASSERT(type.toMIRType() == callee.argTypes[2]); + + MDefinition* ret; + if (!f.emitInstanceCall3(bytecodeOffset, callee, ptr, expected, timeout, + &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitFence(FunctionCompiler& f) { + if (!f.iter().readFence()) { + return false; + } + + f.fence(); + return true; +} + +static bool EmitWake(FunctionCompiler& f) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + LinearMemoryAddress addr; + MDefinition* count; + if (!f.iter().readWake(&addr, &count)) { + return false; + } + + MemoryAccessDesc access(Scalar::Int32, addr.align, addr.offset, + f.bytecodeOffset()); + MDefinition* ptr = f.computeEffectiveAddress(addr.base, &access); + if (!f.inDeadCode() && !ptr) { + return false; + } + + const SymbolicAddressSignature& callee = + f.isNoMemOrMem32() ? SASigWakeM32 : SASigWakeM64; + + MDefinition* ret; + if (!f.emitInstanceCall2(bytecodeOffset, callee, ptr, count, &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitAtomicXchg(FunctionCompiler& f, ValType type, + Scalar::Type viewType) { + LinearMemoryAddress addr; + MDefinition* value; + if (!f.iter().readAtomicRMW(&addr, type, byteSize(viewType), &value)) { + return false; + } + + MemoryAccessDesc access(viewType, addr.align, addr.offset, f.bytecodeOffset(), + Synchronization::Full()); + MDefinition* ins = f.atomicExchangeHeap(addr.base, &access, type, value); + if (!f.inDeadCode() && !ins) { + return false; + } + + f.iter().setResult(ins); + return true; +} + +static bool EmitMemCopyCall(FunctionCompiler& f, MDefinition* dst, + MDefinition* src, MDefinition* len) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* memoryBase = f.memoryBase(); + const SymbolicAddressSignature& callee = + (f.moduleEnv().usesSharedMemory() + ? (f.isMem32() ? SASigMemCopySharedM32 : SASigMemCopySharedM64) + : (f.isMem32() ? SASigMemCopyM32 : SASigMemCopyM64)); + + return f.emitInstanceCall4(bytecodeOffset, callee, dst, src, len, memoryBase); +} + +static bool EmitMemCopyInline(FunctionCompiler& f, MDefinition* dst, + MDefinition* src, uint32_t length) { + MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryCopyLength); + + // Compute the number of copies of each width we will need to do + size_t remainder = length; +#ifdef ENABLE_WASM_SIMD + size_t numCopies16 = 0; + if (MacroAssembler::SupportsFastUnalignedFPAccesses()) { + numCopies16 = remainder / sizeof(V128); + remainder %= sizeof(V128); + } +#endif +#ifdef JS_64BIT + size_t numCopies8 = remainder / sizeof(uint64_t); + remainder %= sizeof(uint64_t); +#endif + size_t numCopies4 = remainder / sizeof(uint32_t); + remainder %= sizeof(uint32_t); + size_t numCopies2 = remainder / sizeof(uint16_t); + remainder %= sizeof(uint16_t); + size_t numCopies1 = remainder; + + // Load all source bytes from low to high using the widest transfer width we + // can for the system. We will trap without writing anything if any source + // byte is out-of-bounds. + size_t offset = 0; + DefVector loadedValues; + +#ifdef ENABLE_WASM_SIMD + for (uint32_t i = 0; i < numCopies16; i++) { + MemoryAccessDesc access(Scalar::Simd128, 1, offset, f.bytecodeOffset()); + auto* load = f.load(src, &access, ValType::V128); + if (!load || !loadedValues.append(load)) { + return false; + } + + offset += sizeof(V128); + } +#endif + +#ifdef JS_64BIT + for (uint32_t i = 0; i < numCopies8; i++) { + MemoryAccessDesc access(Scalar::Int64, 1, offset, f.bytecodeOffset()); + auto* load = f.load(src, &access, ValType::I64); + if (!load || !loadedValues.append(load)) { + return false; + } + + offset += sizeof(uint64_t); + } +#endif + + for (uint32_t i = 0; i < numCopies4; i++) { + MemoryAccessDesc access(Scalar::Uint32, 1, offset, f.bytecodeOffset()); + auto* load = f.load(src, &access, ValType::I32); + if (!load || !loadedValues.append(load)) { + return false; + } + + offset += sizeof(uint32_t); + } + + if (numCopies2) { + MemoryAccessDesc access(Scalar::Uint16, 1, offset, f.bytecodeOffset()); + auto* load = f.load(src, &access, ValType::I32); + if (!load || !loadedValues.append(load)) { + return false; + } + + offset += sizeof(uint16_t); + } + + if (numCopies1) { + MemoryAccessDesc access(Scalar::Uint8, 1, offset, f.bytecodeOffset()); + auto* load = f.load(src, &access, ValType::I32); + if (!load || !loadedValues.append(load)) { + return false; + } + } + + // Store all source bytes to the destination from high to low. We will trap + // without writing anything on the first store if any dest byte is + // out-of-bounds. + offset = length; + + if (numCopies1) { + offset -= sizeof(uint8_t); + + MemoryAccessDesc access(Scalar::Uint8, 1, offset, f.bytecodeOffset()); + auto* value = loadedValues.popCopy(); + f.store(dst, &access, value); + } + + if (numCopies2) { + offset -= sizeof(uint16_t); + + MemoryAccessDesc access(Scalar::Uint16, 1, offset, f.bytecodeOffset()); + auto* value = loadedValues.popCopy(); + f.store(dst, &access, value); + } + + for (uint32_t i = 0; i < numCopies4; i++) { + offset -= sizeof(uint32_t); + + MemoryAccessDesc access(Scalar::Uint32, 1, offset, f.bytecodeOffset()); + auto* value = loadedValues.popCopy(); + f.store(dst, &access, value); + } + +#ifdef JS_64BIT + for (uint32_t i = 0; i < numCopies8; i++) { + offset -= sizeof(uint64_t); + + MemoryAccessDesc access(Scalar::Int64, 1, offset, f.bytecodeOffset()); + auto* value = loadedValues.popCopy(); + f.store(dst, &access, value); + } +#endif + +#ifdef ENABLE_WASM_SIMD + for (uint32_t i = 0; i < numCopies16; i++) { + offset -= sizeof(V128); + + MemoryAccessDesc access(Scalar::Simd128, 1, offset, f.bytecodeOffset()); + auto* value = loadedValues.popCopy(); + f.store(dst, &access, value); + } +#endif + + return true; +} + +static bool EmitMemCopy(FunctionCompiler& f) { + MDefinition *dst, *src, *len; + uint32_t dstMemIndex; + uint32_t srcMemIndex; + if (!f.iter().readMemOrTableCopy(true, &dstMemIndex, &dst, &srcMemIndex, &src, + &len)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + if (len->isConstant()) { + uint64_t length = f.isMem32() ? len->toConstant()->toInt32() + : len->toConstant()->toInt64(); + static_assert(MaxInlineMemoryCopyLength <= UINT32_MAX); + if (length != 0 && length <= MaxInlineMemoryCopyLength) { + return EmitMemCopyInline(f, dst, src, uint32_t(length)); + } + } + + return EmitMemCopyCall(f, dst, src, len); +} + +static bool EmitTableCopy(FunctionCompiler& f) { + MDefinition *dst, *src, *len; + uint32_t dstTableIndex; + uint32_t srcTableIndex; + if (!f.iter().readMemOrTableCopy(false, &dstTableIndex, &dst, &srcTableIndex, + &src, &len)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + MDefinition* dti = f.constantI32(int32_t(dstTableIndex)); + MDefinition* sti = f.constantI32(int32_t(srcTableIndex)); + + return f.emitInstanceCall5(bytecodeOffset, SASigTableCopy, dst, src, len, dti, + sti); +} + +static bool EmitDataOrElemDrop(FunctionCompiler& f, bool isData) { + uint32_t segIndexVal = 0; + if (!f.iter().readDataOrElemDrop(isData, &segIndexVal)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* segIndex = f.constantI32(int32_t(segIndexVal)); + + const SymbolicAddressSignature& callee = + isData ? SASigDataDrop : SASigElemDrop; + return f.emitInstanceCall1(bytecodeOffset, callee, segIndex); +} + +static bool EmitMemFillCall(FunctionCompiler& f, MDefinition* start, + MDefinition* val, MDefinition* len) { + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* memoryBase = f.memoryBase(); + + const SymbolicAddressSignature& callee = + (f.moduleEnv().usesSharedMemory() + ? (f.isMem32() ? SASigMemFillSharedM32 : SASigMemFillSharedM64) + : (f.isMem32() ? SASigMemFillM32 : SASigMemFillM64)); + return f.emitInstanceCall4(bytecodeOffset, callee, start, val, len, + memoryBase); +} + +static bool EmitMemFillInline(FunctionCompiler& f, MDefinition* start, + MDefinition* val, uint32_t length) { + MOZ_ASSERT(length != 0 && length <= MaxInlineMemoryFillLength); + uint32_t value = val->toConstant()->toInt32(); + + // Compute the number of copies of each width we will need to do + size_t remainder = length; +#ifdef ENABLE_WASM_SIMD + size_t numCopies16 = 0; + if (MacroAssembler::SupportsFastUnalignedFPAccesses()) { + numCopies16 = remainder / sizeof(V128); + remainder %= sizeof(V128); + } +#endif +#ifdef JS_64BIT + size_t numCopies8 = remainder / sizeof(uint64_t); + remainder %= sizeof(uint64_t); +#endif + size_t numCopies4 = remainder / sizeof(uint32_t); + remainder %= sizeof(uint32_t); + size_t numCopies2 = remainder / sizeof(uint16_t); + remainder %= sizeof(uint16_t); + size_t numCopies1 = remainder; + + // Generate splatted definitions for wider fills as needed +#ifdef ENABLE_WASM_SIMD + MDefinition* val16 = numCopies16 ? f.constantV128(V128(value)) : nullptr; +#endif +#ifdef JS_64BIT + MDefinition* val8 = + numCopies8 ? f.constantI64(int64_t(SplatByteToUInt(value, 8))) + : nullptr; +#endif + MDefinition* val4 = + numCopies4 ? f.constantI32(int32_t(SplatByteToUInt(value, 4))) + : nullptr; + MDefinition* val2 = + numCopies2 ? f.constantI32(int32_t(SplatByteToUInt(value, 2))) + : nullptr; + + // Store the fill value to the destination from high to low. We will trap + // without writing anything on the first store if any dest byte is + // out-of-bounds. + size_t offset = length; + + if (numCopies1) { + offset -= sizeof(uint8_t); + + MemoryAccessDesc access(Scalar::Uint8, 1, offset, f.bytecodeOffset()); + f.store(start, &access, val); + } + + if (numCopies2) { + offset -= sizeof(uint16_t); + + MemoryAccessDesc access(Scalar::Uint16, 1, offset, f.bytecodeOffset()); + f.store(start, &access, val2); + } + + for (uint32_t i = 0; i < numCopies4; i++) { + offset -= sizeof(uint32_t); + + MemoryAccessDesc access(Scalar::Uint32, 1, offset, f.bytecodeOffset()); + f.store(start, &access, val4); + } + +#ifdef JS_64BIT + for (uint32_t i = 0; i < numCopies8; i++) { + offset -= sizeof(uint64_t); + + MemoryAccessDesc access(Scalar::Int64, 1, offset, f.bytecodeOffset()); + f.store(start, &access, val8); + } +#endif + +#ifdef ENABLE_WASM_SIMD + for (uint32_t i = 0; i < numCopies16; i++) { + offset -= sizeof(V128); + + MemoryAccessDesc access(Scalar::Simd128, 1, offset, f.bytecodeOffset()); + f.store(start, &access, val16); + } +#endif + + return true; +} + +static bool EmitMemFill(FunctionCompiler& f) { + MDefinition *start, *val, *len; + if (!f.iter().readMemFill(&start, &val, &len)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + if (len->isConstant() && val->isConstant()) { + uint64_t length = f.isMem32() ? len->toConstant()->toInt32() + : len->toConstant()->toInt64(); + static_assert(MaxInlineMemoryFillLength <= UINT32_MAX); + if (length != 0 && length <= MaxInlineMemoryFillLength) { + return EmitMemFillInline(f, start, val, uint32_t(length)); + } + } + + return EmitMemFillCall(f, start, val, len); +} + +static bool EmitMemOrTableInit(FunctionCompiler& f, bool isMem) { + uint32_t segIndexVal = 0, dstTableIndex = 0; + MDefinition *dstOff, *srcOff, *len; + if (!f.iter().readMemOrTableInit(isMem, &segIndexVal, &dstTableIndex, &dstOff, + &srcOff, &len)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* segIndex = f.constantI32(int32_t(segIndexVal)); + + if (isMem) { + const SymbolicAddressSignature& callee = + f.isMem32() ? SASigMemInitM32 : SASigMemInitM64; + return f.emitInstanceCall4(bytecodeOffset, callee, dstOff, srcOff, len, + segIndex); + } + + MDefinition* dti = f.constantI32(int32_t(dstTableIndex)); + return f.emitInstanceCall5(bytecodeOffset, SASigTableInit, dstOff, srcOff, + len, segIndex, dti); +} + +// Note, table.{get,grow,set} on table(funcref) are currently rejected by the +// verifier. + +static bool EmitTableFill(FunctionCompiler& f) { + uint32_t tableIndex; + MDefinition *start, *val, *len; + if (!f.iter().readTableFill(&tableIndex, &start, &val, &len)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex)); + if (!tableIndexArg) { + return false; + } + + return f.emitInstanceCall4(bytecodeOffset, SASigTableFill, start, val, len, + tableIndexArg); +} + +#if ENABLE_WASM_MEMORY_CONTROL +static bool EmitMemDiscard(FunctionCompiler& f) { + MDefinition *start, *len; + if (!f.iter().readMemDiscard(&start, &len)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* memoryBase = f.memoryBase(); + + const SymbolicAddressSignature& callee = + (f.moduleEnv().usesSharedMemory() + ? (f.isMem32() ? SASigMemDiscardSharedM32 : SASigMemDiscardSharedM64) + : (f.isMem32() ? SASigMemDiscardM32 : SASigMemDiscardM64)); + return f.emitInstanceCall3(bytecodeOffset, callee, start, len, memoryBase); +} +#endif + +static bool EmitTableGet(FunctionCompiler& f) { + uint32_t tableIndex; + MDefinition* index; + if (!f.iter().readTableGet(&tableIndex, &index)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + const TableDesc& table = f.moduleEnv().tables[tableIndex]; + if (table.elemType.tableRepr() == TableRepr::Ref) { + MDefinition* ret = f.tableGetAnyRef(tableIndex, index); + if (!ret) { + return false; + } + f.iter().setResult(ret); + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex)); + if (!tableIndexArg) { + return false; + } + + // The return value here is either null, denoting an error, or a short-lived + // pointer to a location containing a possibly-null ref. + MDefinition* ret; + if (!f.emitInstanceCall2(bytecodeOffset, SASigTableGet, index, tableIndexArg, + &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitTableGrow(FunctionCompiler& f) { + uint32_t tableIndex; + MDefinition* initValue; + MDefinition* delta; + if (!f.iter().readTableGrow(&tableIndex, &initValue, &delta)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex)); + if (!tableIndexArg) { + return false; + } + + MDefinition* ret; + if (!f.emitInstanceCall3(bytecodeOffset, SASigTableGrow, initValue, delta, + tableIndexArg, &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitTableSet(FunctionCompiler& f) { + uint32_t tableIndex; + MDefinition* index; + MDefinition* value; + if (!f.iter().readTableSet(&tableIndex, &index, &value)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + const TableDesc& table = f.moduleEnv().tables[tableIndex]; + if (table.elemType.tableRepr() == TableRepr::Ref) { + return f.tableSetAnyRef(tableIndex, index, value, bytecodeOffset); + } + + MDefinition* tableIndexArg = f.constantI32(int32_t(tableIndex)); + if (!tableIndexArg) { + return false; + } + + return f.emitInstanceCall3(bytecodeOffset, SASigTableSet, index, value, + tableIndexArg); +} + +static bool EmitTableSize(FunctionCompiler& f) { + uint32_t tableIndex; + if (!f.iter().readTableSize(&tableIndex)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + MDefinition* length = f.loadTableLength(tableIndex); + if (!length) { + return false; + } + + f.iter().setResult(length); + return true; +} + +static bool EmitRefFunc(FunctionCompiler& f) { + uint32_t funcIndex; + if (!f.iter().readRefFunc(&funcIndex)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + + MDefinition* funcIndexArg = f.constantI32(int32_t(funcIndex)); + if (!funcIndexArg) { + return false; + } + + // The return value here is either null, denoting an error, or a short-lived + // pointer to a location containing a possibly-null ref. + MDefinition* ret; + if (!f.emitInstanceCall1(bytecodeOffset, SASigRefFunc, funcIndexArg, &ret)) { + return false; + } + + f.iter().setResult(ret); + return true; +} + +static bool EmitRefNull(FunctionCompiler& f) { + RefType type; + if (!f.iter().readRefNull(&type)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + MDefinition* nullVal = f.constantNullRef(); + if (!nullVal) { + return false; + } + f.iter().setResult(nullVal); + return true; +} + +static bool EmitRefIsNull(FunctionCompiler& f) { + MDefinition* input; + if (!f.iter().readRefIsNull(&input)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + MDefinition* nullVal = f.constantNullRef(); + if (!nullVal) { + return false; + } + f.iter().setResult( + f.compare(input, nullVal, JSOp::Eq, MCompare::Compare_RefOrNull)); + return true; +} + +#ifdef ENABLE_WASM_SIMD +static bool EmitConstSimd128(FunctionCompiler& f) { + V128 v128; + if (!f.iter().readV128Const(&v128)) { + return false; + } + + f.iter().setResult(f.constantV128(v128)); + return true; +} + +static bool EmitBinarySimd128(FunctionCompiler& f, bool commutative, + SimdOp op) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readBinary(ValType::V128, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.binarySimd128(lhs, rhs, commutative, op)); + return true; +} + +static bool EmitTernarySimd128(FunctionCompiler& f, wasm::SimdOp op) { + MDefinition* v0; + MDefinition* v1; + MDefinition* v2; + if (!f.iter().readTernary(ValType::V128, &v0, &v1, &v2)) { + return false; + } + + f.iter().setResult(f.ternarySimd128(v0, v1, v2, op)); + return true; +} + +static bool EmitShiftSimd128(FunctionCompiler& f, SimdOp op) { + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readVectorShift(&lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.shiftSimd128(lhs, rhs, op)); + return true; +} + +static bool EmitSplatSimd128(FunctionCompiler& f, ValType inType, SimdOp op) { + MDefinition* src; + if (!f.iter().readConversion(inType, ValType::V128, &src)) { + return false; + } + + f.iter().setResult(f.scalarToSimd128(src, op)); + return true; +} + +static bool EmitUnarySimd128(FunctionCompiler& f, SimdOp op) { + MDefinition* src; + if (!f.iter().readUnary(ValType::V128, &src)) { + return false; + } + + f.iter().setResult(f.unarySimd128(src, op)); + return true; +} + +static bool EmitReduceSimd128(FunctionCompiler& f, SimdOp op) { + MDefinition* src; + if (!f.iter().readConversion(ValType::V128, ValType::I32, &src)) { + return false; + } + + f.iter().setResult(f.reduceSimd128(src, op, ValType::I32)); + return true; +} + +static bool EmitExtractLaneSimd128(FunctionCompiler& f, ValType outType, + uint32_t laneLimit, SimdOp op) { + uint32_t laneIndex; + MDefinition* src; + if (!f.iter().readExtractLane(outType, laneLimit, &laneIndex, &src)) { + return false; + } + + f.iter().setResult(f.reduceSimd128(src, op, outType, laneIndex)); + return true; +} + +static bool EmitReplaceLaneSimd128(FunctionCompiler& f, ValType laneType, + uint32_t laneLimit, SimdOp op) { + uint32_t laneIndex; + MDefinition* lhs; + MDefinition* rhs; + if (!f.iter().readReplaceLane(laneType, laneLimit, &laneIndex, &lhs, &rhs)) { + return false; + } + + f.iter().setResult(f.replaceLaneSimd128(lhs, rhs, laneIndex, op)); + return true; +} + +static bool EmitShuffleSimd128(FunctionCompiler& f) { + MDefinition* v1; + MDefinition* v2; + V128 control; + if (!f.iter().readVectorShuffle(&v1, &v2, &control)) { + return false; + } + + f.iter().setResult(f.shuffleSimd128(v1, v2, control)); + return true; +} + +static bool EmitLoadSplatSimd128(FunctionCompiler& f, Scalar::Type viewType, + wasm::SimdOp splatOp) { + LinearMemoryAddress addr; + if (!f.iter().readLoadSplat(Scalar::byteSize(viewType), &addr)) { + return false; + } + + f.iter().setResult(f.loadSplatSimd128(viewType, addr, splatOp)); + return true; +} + +static bool EmitLoadExtendSimd128(FunctionCompiler& f, wasm::SimdOp op) { + LinearMemoryAddress addr; + if (!f.iter().readLoadExtend(&addr)) { + return false; + } + + f.iter().setResult(f.loadExtendSimd128(addr, op)); + return true; +} + +static bool EmitLoadZeroSimd128(FunctionCompiler& f, Scalar::Type viewType, + size_t numBytes) { + LinearMemoryAddress addr; + if (!f.iter().readLoadSplat(numBytes, &addr)) { + return false; + } + + f.iter().setResult(f.loadZeroSimd128(viewType, numBytes, addr)); + return true; +} + +static bool EmitLoadLaneSimd128(FunctionCompiler& f, uint32_t laneSize) { + uint32_t laneIndex; + MDefinition* src; + LinearMemoryAddress addr; + if (!f.iter().readLoadLane(laneSize, &addr, &laneIndex, &src)) { + return false; + } + + f.iter().setResult(f.loadLaneSimd128(laneSize, addr, laneIndex, src)); + return true; +} + +static bool EmitStoreLaneSimd128(FunctionCompiler& f, uint32_t laneSize) { + uint32_t laneIndex; + MDefinition* src; + LinearMemoryAddress addr; + if (!f.iter().readStoreLane(laneSize, &addr, &laneIndex, &src)) { + return false; + } + + f.storeLaneSimd128(laneSize, addr, laneIndex, src); + return true; +} + +#endif // ENABLE_WASM_SIMD + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +static bool EmitRefAsNonNull(FunctionCompiler& f) { + MDefinition* value; + if (!f.iter().readRefAsNonNull(&value)) { + return false; + } + + return f.refAsNonNull(value); +} + +static bool EmitBrOnNull(FunctionCompiler& f) { + uint32_t relativeDepth; + ResultType type; + DefVector values; + MDefinition* condition; + if (!f.iter().readBrOnNull(&relativeDepth, &type, &values, &condition)) { + return false; + } + + return f.brOnNull(relativeDepth, values, type, condition); +} + +static bool EmitBrOnNonNull(FunctionCompiler& f) { + uint32_t relativeDepth; + ResultType type; + DefVector values; + MDefinition* condition; + if (!f.iter().readBrOnNonNull(&relativeDepth, &type, &values, &condition)) { + return false; + } + + return f.brOnNonNull(relativeDepth, values, type, condition); +} + +static bool EmitCallRef(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + const FuncType* funcType; + MDefinition* callee; + DefVector args; + + if (!f.iter().readCallRef(&funcType, &callee, &args)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + CallCompileState call; + if (!EmitCallArgs(f, *funcType, args, &call)) { + return false; + } + + DefVector results; + if (!f.callRef(*funcType, callee, lineOrBytecode, call, &results)) { + return false; + } + + f.iter().setResults(results.length(), results); + return true; +} + +#endif // ENABLE_WASM_FUNCTION_REFERENCES + +#ifdef ENABLE_WASM_GC + +static bool EmitStructNew(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex; + DefVector args; + if (!f.iter().readStructNew(&typeIndex, &args)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType(); + MOZ_ASSERT(args.length() == structType.fields_.length()); + + // Allocate a default initialized struct. This requires the type definition + // for the struct. + MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex); + if (!typeDefData) { + return false; + } + + // Create call: structObject = Instance::structNewUninit(typeDefData) + MDefinition* structObject; + if (!f.emitInstanceCall1(lineOrBytecode, SASigStructNewUninit, typeDefData, + &structObject)) { + return false; + } + + // And fill in the fields. + for (uint32_t fieldIndex = 0; fieldIndex < structType.fields_.length(); + fieldIndex++) { + if (!f.mirGen().ensureBallast()) { + return false; + } + const StructField& field = structType.fields_[fieldIndex]; + if (!f.writeValueToStructField(lineOrBytecode, field, structObject, + args[fieldIndex], + WasmPreBarrierKind::None)) { + return false; + } + } + + f.iter().setResult(structObject); + return true; +} + +static bool EmitStructNewDefault(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex; + if (!f.iter().readStructNewDefault(&typeIndex)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Allocate a default initialized struct. This requires the type definition + // for the struct. + MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex); + if (!typeDefData) { + return false; + } + + // Create call: structObject = Instance::structNew(typeDefData) + MDefinition* structObject; + if (!f.emitInstanceCall1(lineOrBytecode, SASigStructNew, typeDefData, + &structObject)) { + return false; + } + + f.iter().setResult(structObject); + return true; +} + +static bool EmitStructSet(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex; + uint32_t fieldIndex; + MDefinition* structObject; + MDefinition* value; + if (!f.iter().readStructSet(&typeIndex, &fieldIndex, &structObject, &value)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Check for null is done at writeValueToStructField. + + // And fill in the field. + const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType(); + const StructField& field = structType.fields_[fieldIndex]; + return f.writeValueToStructField(lineOrBytecode, field, structObject, value, + WasmPreBarrierKind::Normal); +} + +static bool EmitStructGet(FunctionCompiler& f, FieldWideningOp wideningOp) { + uint32_t typeIndex; + uint32_t fieldIndex; + MDefinition* structObject; + if (!f.iter().readStructGet(&typeIndex, &fieldIndex, wideningOp, + &structObject)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Check for null is done at readValueFromStructField. + + // And fetch the data. + const StructType& structType = (*f.moduleEnv().types)[typeIndex].structType(); + const StructField& field = structType.fields_[fieldIndex]; + MDefinition* load = + f.readValueFromStructField(field, wideningOp, structObject); + if (!load) { + return false; + } + + f.iter().setResult(load); + return true; +} + +static bool EmitArrayNew(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex; + MDefinition* numElements; + MDefinition* fillValue; + if (!f.iter().readArrayNew(&typeIndex, &numElements, &fillValue)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by + // this helper will trap. + MDefinition* arrayObject = f.createArrayNewCallAndLoop( + lineOrBytecode, typeIndex, numElements, fillValue); + if (!arrayObject) { + return false; + } + + f.iter().setResult(arrayObject); + return true; +} + +static bool EmitArrayNewDefault(FunctionCompiler& f) { + // This is almost identical to EmitArrayNew, except we skip the + // initialisation loop. + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex; + MDefinition* numElements; + if (!f.iter().readArrayNewDefault(&typeIndex, &numElements)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Create the array object, default-initialized. + MDefinition* arrayObject = f.createDefaultInitializedArrayObject( + lineOrBytecode, typeIndex, numElements); + if (!arrayObject) { + return false; + } + + f.iter().setResult(arrayObject); + return true; +} + +static bool EmitArrayNewFixed(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex, numElements; + DefVector values; + + if (!f.iter().readArrayNewFixed(&typeIndex, &numElements, &values)) { + return false; + } + MOZ_ASSERT(values.length() == numElements); + + if (f.inDeadCode()) { + return true; + } + + MDefinition* numElementsDef = f.constantI32(int32_t(numElements)); + if (!numElementsDef) { + return false; + } + + // Create the array object, default-initialized. + MDefinition* arrayObject = f.createDefaultInitializedArrayObject( + lineOrBytecode, typeIndex, numElementsDef); + if (!arrayObject) { + return false; + } + + // Make `base` point at the first byte of the (OOL) data area. + MDefinition* base = f.getWasmArrayObjectData(arrayObject); + if (!base) { + return false; + } + + // Write each element in turn. + const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType(); + FieldType elemFieldType = arrayType.elementType_; + uint32_t elemSize = elemFieldType.size(); + + // How do we know that the offset expression `i * elemSize` below remains + // within 2^31 (signed-i32) range? In the worst case we will have 16-byte + // values, and there can be at most MaxFunctionBytes expressions, if it were + // theoretically possible to generate one expression per instruction byte. + // Hence the max offset we can be expected to generate is + // `16 * MaxFunctionBytes`. + static_assert(16 /* sizeof v128 */ * MaxFunctionBytes <= + MaxArrayPayloadBytes); + MOZ_RELEASE_ASSERT(numElements <= MaxFunctionBytes); + + for (uint32_t i = 0; i < numElements; i++) { + if (!f.mirGen().ensureBallast()) { + return false; + } + // `i * elemSize` is made safe by the assertions above. + if (!f.writeGcValueAtBasePlusOffset( + lineOrBytecode, elemFieldType, arrayObject, + AliasSet::WasmArrayDataArea, values[numElements - 1 - i], base, + i * elemSize, false, WasmPreBarrierKind::None)) { + return false; + } + } + + f.iter().setResult(arrayObject); + return true; +} + +static bool EmitArrayNewData(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex, segIndex; + MDefinition* segByteOffset; + MDefinition* numElements; + if (!f.iter().readArrayNewData(&typeIndex, &segIndex, &segByteOffset, + &numElements)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Get the type definition data for the array as a whole. + MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex); + if (!typeDefData) { + return false; + } + + // Other values we need to pass to the instance call: + MDefinition* segIndexM = f.constantI32(int32_t(segIndex)); + if (!segIndexM) { + return false; + } + + // Create call: + // arrayObject = Instance::arrayNewData(segByteOffset:u32, numElements:u32, + // typeDefData:word, segIndex:u32) + // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by + // this call will trap. + MDefinition* arrayObject; + if (!f.emitInstanceCall4(lineOrBytecode, SASigArrayNewData, segByteOffset, + numElements, typeDefData, segIndexM, &arrayObject)) { + return false; + } + + f.iter().setResult(arrayObject); + return true; +} + +static bool EmitArrayNewElem(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex, segIndex; + MDefinition* segElemIndex; + MDefinition* numElements; + if (!f.iter().readArrayNewElem(&typeIndex, &segIndex, &segElemIndex, + &numElements)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Get the type definition for the array as a whole. + // Get the type definition data for the array as a whole. + MDefinition* typeDefData = f.loadTypeDefInstanceData(typeIndex); + if (!typeDefData) { + return false; + } + + // Other values we need to pass to the instance call: + MDefinition* segIndexM = f.constantI32(int32_t(segIndex)); + if (!segIndexM) { + return false; + } + + // Create call: + // arrayObject = Instance::arrayNewElem(segElemIndex:u32, numElements:u32, + // typeDefData:word, segIndex:u32) + // If the requested size exceeds MaxArrayPayloadBytes, the MIR generated by + // this call will trap. + MDefinition* arrayObject; + if (!f.emitInstanceCall4(lineOrBytecode, SASigArrayNewElem, segElemIndex, + numElements, typeDefData, segIndexM, &arrayObject)) { + return false; + } + + f.iter().setResult(arrayObject); + return true; +} + +static bool EmitArraySet(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + uint32_t typeIndex; + MDefinition* value; + MDefinition* index; + MDefinition* arrayObject; + if (!f.iter().readArraySet(&typeIndex, &value, &index, &arrayObject)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Check for null is done at setupForArrayAccess. + + // Create the object null check and the array bounds check and get the OOL + // data pointer. + MDefinition* base = f.setupForArrayAccess(arrayObject, index); + if (!base) { + return false; + } + + // And do the store. + const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType(); + FieldType elemFieldType = arrayType.elementType_; + uint32_t elemSize = elemFieldType.size(); + MOZ_ASSERT(elemSize >= 1 && elemSize <= 16); + + return f.writeGcValueAtBasePlusScaledIndex( + lineOrBytecode, elemFieldType, arrayObject, AliasSet::WasmArrayDataArea, + value, base, elemSize, index, WasmPreBarrierKind::Normal); +} + +static bool EmitArrayGet(FunctionCompiler& f, FieldWideningOp wideningOp) { + uint32_t typeIndex; + MDefinition* index; + MDefinition* arrayObject; + if (!f.iter().readArrayGet(&typeIndex, wideningOp, &index, &arrayObject)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Check for null is done at setupForArrayAccess. + + // Create the object null check and the array bounds check and get the OOL + // data pointer. + MDefinition* base = f.setupForArrayAccess(arrayObject, index); + if (!base) { + return false; + } + + // And do the load. + const ArrayType& arrayType = (*f.moduleEnv().types)[typeIndex].arrayType(); + FieldType elemFieldType = arrayType.elementType_; + uint32_t elemSize = elemFieldType.size(); + MOZ_ASSERT(elemSize >= 1 && elemSize <= 16); + + MDefinition* load = f.readGcValueAtBasePlusScaledIndex( + elemFieldType, wideningOp, arrayObject, AliasSet::WasmArrayDataArea, base, + elemSize, index); + if (!load) { + return false; + } + + f.iter().setResult(load); + return true; +} + +static bool EmitArrayLen(FunctionCompiler& f, bool decodeIgnoredTypeIndex) { + MDefinition* arrayObject; + if (!f.iter().readArrayLen(decodeIgnoredTypeIndex, &arrayObject)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + // Check for null is done at getWasmArrayObjectNumElements. + + // Get the size value for the array + MDefinition* numElements = f.getWasmArrayObjectNumElements(arrayObject); + if (!numElements) { + return false; + } + + f.iter().setResult(numElements); + return true; +} + +static bool EmitArrayCopy(FunctionCompiler& f) { + uint32_t lineOrBytecode = f.readCallSiteLineOrBytecode(); + + int32_t elemSize; + bool elemsAreRefTyped; + MDefinition* dstArrayObject; + MDefinition* dstArrayIndex; + MDefinition* srcArrayObject; + MDefinition* srcArrayIndex; + MDefinition* numElements; + if (!f.iter().readArrayCopy(&elemSize, &elemsAreRefTyped, &dstArrayObject, + &dstArrayIndex, &srcArrayObject, &srcArrayIndex, + &numElements)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + MOZ_ASSERT_IF(elemsAreRefTyped, + size_t(elemSize) == MIRTypeToSize(TargetWordMIRType())); + MOZ_ASSERT_IF(!elemsAreRefTyped, elemSize == 1 || elemSize == 2 || + elemSize == 4 || elemSize == 8 || + elemSize == 16); + + // A negative element size is used to inform Instance::arrayCopy that the + // values are reftyped. This avoids having to pass it an extra boolean + // argument. + MDefinition* elemSizeDef = + f.constantI32(elemsAreRefTyped ? -elemSize : elemSize); + if (!elemSizeDef) { + return false; + } + + // Create call: + // Instance::arrayCopy(dstArrayObject:word, dstArrayIndex:u32, + // srcArrayObject:word, srcArrayIndex:u32, + // numElements:u32, + // (elemsAreRefTyped ? -elemSize : elemSize):u32)) + return f.emitInstanceCall6(lineOrBytecode, SASigArrayCopy, dstArrayObject, + dstArrayIndex, srcArrayObject, srcArrayIndex, + numElements, elemSizeDef); +} + +static bool EmitRefTestV5(FunctionCompiler& f) { + MDefinition* ref; + RefType sourceType; + uint32_t typeIndex; + if (!f.iter().readRefTestV5(&sourceType, &typeIndex, &ref)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + const TypeDef& typeDef = f.moduleEnv().types->type(typeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, false); + MDefinition* success = f.refTest(ref, sourceType, destType); + if (!success) { + return false; + } + + f.iter().setResult(success); + return true; +} + +static bool EmitRefCastV5(FunctionCompiler& f) { + MDefinition* ref; + RefType sourceType; + uint32_t typeIndex; + if (!f.iter().readRefCastV5(&sourceType, &typeIndex, &ref)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + const TypeDef& typeDef = f.moduleEnv().types->type(typeIndex); + RefType destType = RefType::fromTypeDef(&typeDef, /*nullable=*/true); + if (!f.refCast(ref, sourceType, destType)) { + return false; + } + + f.iter().setResult(ref); + return true; +} + +static bool EmitRefTest(FunctionCompiler& f, bool nullable) { + MDefinition* ref; + RefType sourceType; + RefType destType; + if (!f.iter().readRefTest(nullable, &sourceType, &destType, &ref)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + MDefinition* success = f.refTest(ref, sourceType, destType); + if (!success) { + return false; + } + + f.iter().setResult(success); + return true; +} + +static bool EmitRefCast(FunctionCompiler& f, bool nullable) { + MDefinition* ref; + RefType sourceType; + RefType destType; + if (!f.iter().readRefCast(nullable, &sourceType, &destType, &ref)) { + return false; + } + + if (f.inDeadCode()) { + return true; + } + + if (!f.refCast(ref, sourceType, destType)) { + return false; + } + + f.iter().setResult(ref); + return true; +} + +static bool EmitBrOnCast(FunctionCompiler& f) { + bool onSuccess; + uint32_t labelRelativeDepth; + RefType sourceType; + RefType destType; + ResultType labelType; + DefVector values; + if (!f.iter().readBrOnCast(&onSuccess, &labelRelativeDepth, &sourceType, + &destType, &labelType, &values)) { + return false; + } + + return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, destType, + labelType, values); +} + +static bool EmitBrOnCastCommonV5(FunctionCompiler& f, bool onSuccess) { + uint32_t labelRelativeDepth; + RefType sourceType; + uint32_t castTypeIndex; + ResultType labelType; + DefVector values; + if (onSuccess + ? !f.iter().readBrOnCastV5(&labelRelativeDepth, &sourceType, + &castTypeIndex, &labelType, &values) + : !f.iter().readBrOnCastFailV5(&labelRelativeDepth, &sourceType, + &castTypeIndex, &labelType, &values)) { + return false; + } + + const TypeDef& typeDef = f.moduleEnv().types->type(castTypeIndex); + RefType type = RefType::fromTypeDef(&typeDef, false); + return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, type, + labelType, values); +} + +static bool EmitBrOnCastHeapV5(FunctionCompiler& f, bool onSuccess, + bool nullable) { + uint32_t labelRelativeDepth; + RefType sourceType; + RefType destType; + ResultType labelType; + DefVector values; + if (onSuccess ? !f.iter().readBrOnCastHeapV5(nullable, &labelRelativeDepth, + &sourceType, &destType, + &labelType, &values) + : !f.iter().readBrOnCastFailHeapV5( + nullable, &labelRelativeDepth, &sourceType, &destType, + &labelType, &values)) { + return false; + } + + return f.brOnCastCommon(onSuccess, labelRelativeDepth, sourceType, destType, + labelType, values); +} + +static bool EmitRefAsStructV5(FunctionCompiler& f) { + MDefinition* value; + if (!f.iter().readConversion(ValType(RefType::any()), + ValType(RefType::struct_().asNonNullable()), + &value)) { + return false; + } + f.iter().setResult(value); + return true; +} + +static bool EmitBrOnNonStructV5(FunctionCompiler& f) { + uint32_t labelRelativeDepth; + ResultType labelType; + DefVector values; + if (!f.iter().readBrOnNonStructV5(&labelRelativeDepth, &labelType, &values)) { + return false; + } + return f.brOnNonStruct(values); +} + +static bool EmitExternInternalize(FunctionCompiler& f) { + // extern.internalize is a no-op because anyref and extern share the same + // representation + MDefinition* ref; + if (!f.iter().readRefConversion(RefType::extern_(), RefType::any(), &ref)) { + return false; + } + + f.iter().setResult(ref); + return true; +} + +static bool EmitExternExternalize(FunctionCompiler& f) { + // extern.externalize is a no-op because anyref and extern share the same + // representation + MDefinition* ref; + if (!f.iter().readRefConversion(RefType::any(), RefType::extern_(), &ref)) { + return false; + } + + f.iter().setResult(ref); + return true; +} + +#endif // ENABLE_WASM_GC + +static bool EmitIntrinsic(FunctionCompiler& f) { + // It's almost possible to use FunctionCompiler::emitInstanceCallN here. + // Unfortunately not currently possible though, since ::emitInstanceCallN + // expects an array of arguments along with a size, and that's not what is + // available here. It would be possible if we were prepared to copy + // `intrinsic->params` into a fixed-sized (16 element?) array, add + // `memoryBase`, and make the call. + const Intrinsic* intrinsic; + + DefVector params; + if (!f.iter().readIntrinsic(&intrinsic, ¶ms)) { + return false; + } + + uint32_t bytecodeOffset = f.readBytecodeOffset(); + const SymbolicAddressSignature& callee = intrinsic->signature; + + CallCompileState args; + if (!f.passInstance(callee.argTypes[0], &args)) { + return false; + } + + if (!f.passArgs(params, intrinsic->params, &args)) { + return false; + } + + MDefinition* memoryBase = f.memoryBase(); + if (!f.passArg(memoryBase, MIRType::Pointer, &args)) { + return false; + } + + if (!f.finishCall(&args)) { + return false; + } + + return f.builtinInstanceMethodCall(callee, bytecodeOffset, args); +} + +static bool EmitBodyExprs(FunctionCompiler& f) { + if (!f.iter().startFunction(f.funcIndex(), f.locals())) { + return false; + } + +#define CHECK(c) \ + if (!(c)) return false; \ + break + + while (true) { + if (!f.mirGen().ensureBallast()) { + return false; + } + + OpBytes op; + if (!f.iter().readOp(&op)) { + return false; + } + + switch (op.b0) { + case uint16_t(Op::End): + if (!EmitEnd(f)) { + return false; + } + if (f.iter().controlStackEmpty()) { + return true; + } + break; + + // Control opcodes + case uint16_t(Op::Unreachable): + CHECK(EmitUnreachable(f)); + case uint16_t(Op::Nop): + CHECK(f.iter().readNop()); + case uint16_t(Op::Block): + CHECK(EmitBlock(f)); + case uint16_t(Op::Loop): + CHECK(EmitLoop(f)); + case uint16_t(Op::If): + CHECK(EmitIf(f)); + case uint16_t(Op::Else): + CHECK(EmitElse(f)); + case uint16_t(Op::Try): + if (!f.moduleEnv().exceptionsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitTry(f)); + case uint16_t(Op::Catch): + if (!f.moduleEnv().exceptionsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitCatch(f)); + case uint16_t(Op::CatchAll): + if (!f.moduleEnv().exceptionsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitCatchAll(f)); + case uint16_t(Op::Delegate): + if (!f.moduleEnv().exceptionsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + if (!EmitDelegate(f)) { + return false; + } + break; + case uint16_t(Op::Throw): + if (!f.moduleEnv().exceptionsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitThrow(f)); + case uint16_t(Op::Rethrow): + if (!f.moduleEnv().exceptionsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitRethrow(f)); + case uint16_t(Op::Br): + CHECK(EmitBr(f)); + case uint16_t(Op::BrIf): + CHECK(EmitBrIf(f)); + case uint16_t(Op::BrTable): + CHECK(EmitBrTable(f)); + case uint16_t(Op::Return): + CHECK(EmitReturn(f)); + + // Calls + case uint16_t(Op::Call): + CHECK(EmitCall(f, /* asmJSFuncDef = */ false)); + case uint16_t(Op::CallIndirect): + CHECK(EmitCallIndirect(f, /* oldStyle = */ false)); + + // Parametric operators + case uint16_t(Op::Drop): + CHECK(f.iter().readDrop()); + case uint16_t(Op::SelectNumeric): + CHECK(EmitSelect(f, /*typed*/ false)); + case uint16_t(Op::SelectTyped): + CHECK(EmitSelect(f, /*typed*/ true)); + + // Locals and globals + case uint16_t(Op::LocalGet): + CHECK(EmitGetLocal(f)); + case uint16_t(Op::LocalSet): + CHECK(EmitSetLocal(f)); + case uint16_t(Op::LocalTee): + CHECK(EmitTeeLocal(f)); + case uint16_t(Op::GlobalGet): + CHECK(EmitGetGlobal(f)); + case uint16_t(Op::GlobalSet): + CHECK(EmitSetGlobal(f)); + case uint16_t(Op::TableGet): + CHECK(EmitTableGet(f)); + case uint16_t(Op::TableSet): + CHECK(EmitTableSet(f)); + + // Memory-related operators + case uint16_t(Op::I32Load): + CHECK(EmitLoad(f, ValType::I32, Scalar::Int32)); + case uint16_t(Op::I64Load): + CHECK(EmitLoad(f, ValType::I64, Scalar::Int64)); + case uint16_t(Op::F32Load): + CHECK(EmitLoad(f, ValType::F32, Scalar::Float32)); + case uint16_t(Op::F64Load): + CHECK(EmitLoad(f, ValType::F64, Scalar::Float64)); + case uint16_t(Op::I32Load8S): + CHECK(EmitLoad(f, ValType::I32, Scalar::Int8)); + case uint16_t(Op::I32Load8U): + CHECK(EmitLoad(f, ValType::I32, Scalar::Uint8)); + case uint16_t(Op::I32Load16S): + CHECK(EmitLoad(f, ValType::I32, Scalar::Int16)); + case uint16_t(Op::I32Load16U): + CHECK(EmitLoad(f, ValType::I32, Scalar::Uint16)); + case uint16_t(Op::I64Load8S): + CHECK(EmitLoad(f, ValType::I64, Scalar::Int8)); + case uint16_t(Op::I64Load8U): + CHECK(EmitLoad(f, ValType::I64, Scalar::Uint8)); + case uint16_t(Op::I64Load16S): + CHECK(EmitLoad(f, ValType::I64, Scalar::Int16)); + case uint16_t(Op::I64Load16U): + CHECK(EmitLoad(f, ValType::I64, Scalar::Uint16)); + case uint16_t(Op::I64Load32S): + CHECK(EmitLoad(f, ValType::I64, Scalar::Int32)); + case uint16_t(Op::I64Load32U): + CHECK(EmitLoad(f, ValType::I64, Scalar::Uint32)); + case uint16_t(Op::I32Store): + CHECK(EmitStore(f, ValType::I32, Scalar::Int32)); + case uint16_t(Op::I64Store): + CHECK(EmitStore(f, ValType::I64, Scalar::Int64)); + case uint16_t(Op::F32Store): + CHECK(EmitStore(f, ValType::F32, Scalar::Float32)); + case uint16_t(Op::F64Store): + CHECK(EmitStore(f, ValType::F64, Scalar::Float64)); + case uint16_t(Op::I32Store8): + CHECK(EmitStore(f, ValType::I32, Scalar::Int8)); + case uint16_t(Op::I32Store16): + CHECK(EmitStore(f, ValType::I32, Scalar::Int16)); + case uint16_t(Op::I64Store8): + CHECK(EmitStore(f, ValType::I64, Scalar::Int8)); + case uint16_t(Op::I64Store16): + CHECK(EmitStore(f, ValType::I64, Scalar::Int16)); + case uint16_t(Op::I64Store32): + CHECK(EmitStore(f, ValType::I64, Scalar::Int32)); + case uint16_t(Op::MemorySize): + CHECK(EmitMemorySize(f)); + case uint16_t(Op::MemoryGrow): + CHECK(EmitMemoryGrow(f)); + + // Constants + case uint16_t(Op::I32Const): + CHECK(EmitI32Const(f)); + case uint16_t(Op::I64Const): + CHECK(EmitI64Const(f)); + case uint16_t(Op::F32Const): + CHECK(EmitF32Const(f)); + case uint16_t(Op::F64Const): + CHECK(EmitF64Const(f)); + + // Comparison operators + case uint16_t(Op::I32Eqz): + CHECK(EmitConversion(f, ValType::I32, ValType::I32)); + case uint16_t(Op::I32Eq): + CHECK( + EmitComparison(f, ValType::I32, JSOp::Eq, MCompare::Compare_Int32)); + case uint16_t(Op::I32Ne): + CHECK( + EmitComparison(f, ValType::I32, JSOp::Ne, MCompare::Compare_Int32)); + case uint16_t(Op::I32LtS): + CHECK( + EmitComparison(f, ValType::I32, JSOp::Lt, MCompare::Compare_Int32)); + case uint16_t(Op::I32LtU): + CHECK(EmitComparison(f, ValType::I32, JSOp::Lt, + MCompare::Compare_UInt32)); + case uint16_t(Op::I32GtS): + CHECK( + EmitComparison(f, ValType::I32, JSOp::Gt, MCompare::Compare_Int32)); + case uint16_t(Op::I32GtU): + CHECK(EmitComparison(f, ValType::I32, JSOp::Gt, + MCompare::Compare_UInt32)); + case uint16_t(Op::I32LeS): + CHECK( + EmitComparison(f, ValType::I32, JSOp::Le, MCompare::Compare_Int32)); + case uint16_t(Op::I32LeU): + CHECK(EmitComparison(f, ValType::I32, JSOp::Le, + MCompare::Compare_UInt32)); + case uint16_t(Op::I32GeS): + CHECK( + EmitComparison(f, ValType::I32, JSOp::Ge, MCompare::Compare_Int32)); + case uint16_t(Op::I32GeU): + CHECK(EmitComparison(f, ValType::I32, JSOp::Ge, + MCompare::Compare_UInt32)); + case uint16_t(Op::I64Eqz): + CHECK(EmitConversion(f, ValType::I64, ValType::I32)); + case uint16_t(Op::I64Eq): + CHECK( + EmitComparison(f, ValType::I64, JSOp::Eq, MCompare::Compare_Int64)); + case uint16_t(Op::I64Ne): + CHECK( + EmitComparison(f, ValType::I64, JSOp::Ne, MCompare::Compare_Int64)); + case uint16_t(Op::I64LtS): + CHECK( + EmitComparison(f, ValType::I64, JSOp::Lt, MCompare::Compare_Int64)); + case uint16_t(Op::I64LtU): + CHECK(EmitComparison(f, ValType::I64, JSOp::Lt, + MCompare::Compare_UInt64)); + case uint16_t(Op::I64GtS): + CHECK( + EmitComparison(f, ValType::I64, JSOp::Gt, MCompare::Compare_Int64)); + case uint16_t(Op::I64GtU): + CHECK(EmitComparison(f, ValType::I64, JSOp::Gt, + MCompare::Compare_UInt64)); + case uint16_t(Op::I64LeS): + CHECK( + EmitComparison(f, ValType::I64, JSOp::Le, MCompare::Compare_Int64)); + case uint16_t(Op::I64LeU): + CHECK(EmitComparison(f, ValType::I64, JSOp::Le, + MCompare::Compare_UInt64)); + case uint16_t(Op::I64GeS): + CHECK( + EmitComparison(f, ValType::I64, JSOp::Ge, MCompare::Compare_Int64)); + case uint16_t(Op::I64GeU): + CHECK(EmitComparison(f, ValType::I64, JSOp::Ge, + MCompare::Compare_UInt64)); + case uint16_t(Op::F32Eq): + CHECK(EmitComparison(f, ValType::F32, JSOp::Eq, + MCompare::Compare_Float32)); + case uint16_t(Op::F32Ne): + CHECK(EmitComparison(f, ValType::F32, JSOp::Ne, + MCompare::Compare_Float32)); + case uint16_t(Op::F32Lt): + CHECK(EmitComparison(f, ValType::F32, JSOp::Lt, + MCompare::Compare_Float32)); + case uint16_t(Op::F32Gt): + CHECK(EmitComparison(f, ValType::F32, JSOp::Gt, + MCompare::Compare_Float32)); + case uint16_t(Op::F32Le): + CHECK(EmitComparison(f, ValType::F32, JSOp::Le, + MCompare::Compare_Float32)); + case uint16_t(Op::F32Ge): + CHECK(EmitComparison(f, ValType::F32, JSOp::Ge, + MCompare::Compare_Float32)); + case uint16_t(Op::F64Eq): + CHECK(EmitComparison(f, ValType::F64, JSOp::Eq, + MCompare::Compare_Double)); + case uint16_t(Op::F64Ne): + CHECK(EmitComparison(f, ValType::F64, JSOp::Ne, + MCompare::Compare_Double)); + case uint16_t(Op::F64Lt): + CHECK(EmitComparison(f, ValType::F64, JSOp::Lt, + MCompare::Compare_Double)); + case uint16_t(Op::F64Gt): + CHECK(EmitComparison(f, ValType::F64, JSOp::Gt, + MCompare::Compare_Double)); + case uint16_t(Op::F64Le): + CHECK(EmitComparison(f, ValType::F64, JSOp::Le, + MCompare::Compare_Double)); + case uint16_t(Op::F64Ge): + CHECK(EmitComparison(f, ValType::F64, JSOp::Ge, + MCompare::Compare_Double)); + + // Numeric operators + case uint16_t(Op::I32Clz): + CHECK(EmitUnaryWithType(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32Ctz): + CHECK(EmitUnaryWithType(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32Popcnt): + CHECK(EmitUnaryWithType(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32Add): + CHECK(EmitAdd(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32Sub): + CHECK(EmitSub(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32Mul): + CHECK(EmitMul(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32DivS): + case uint16_t(Op::I32DivU): + CHECK( + EmitDiv(f, ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32DivU)); + case uint16_t(Op::I32RemS): + case uint16_t(Op::I32RemU): + CHECK( + EmitRem(f, ValType::I32, MIRType::Int32, Op(op.b0) == Op::I32RemU)); + case uint16_t(Op::I32And): + CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32, + MWasmBinaryBitwise::SubOpcode::And)); + case uint16_t(Op::I32Or): + CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32, + MWasmBinaryBitwise::SubOpcode::Or)); + case uint16_t(Op::I32Xor): + CHECK(EmitBitwiseAndOrXor(f, ValType::I32, MIRType::Int32, + MWasmBinaryBitwise::SubOpcode::Xor)); + case uint16_t(Op::I32Shl): + CHECK(EmitShift(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32ShrS): + CHECK(EmitShift(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32ShrU): + CHECK(EmitUrsh(f, ValType::I32, MIRType::Int32)); + case uint16_t(Op::I32Rotl): + case uint16_t(Op::I32Rotr): + CHECK(EmitRotate(f, ValType::I32, Op(op.b0) == Op::I32Rotl)); + case uint16_t(Op::I64Clz): + CHECK(EmitUnaryWithType(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64Ctz): + CHECK(EmitUnaryWithType(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64Popcnt): + CHECK(EmitUnaryWithType(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64Add): + CHECK(EmitAdd(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64Sub): + CHECK(EmitSub(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64Mul): + CHECK(EmitMul(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64DivS): + case uint16_t(Op::I64DivU): + CHECK( + EmitDiv(f, ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64DivU)); + case uint16_t(Op::I64RemS): + case uint16_t(Op::I64RemU): + CHECK( + EmitRem(f, ValType::I64, MIRType::Int64, Op(op.b0) == Op::I64RemU)); + case uint16_t(Op::I64And): + CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64, + MWasmBinaryBitwise::SubOpcode::And)); + case uint16_t(Op::I64Or): + CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64, + MWasmBinaryBitwise::SubOpcode::Or)); + case uint16_t(Op::I64Xor): + CHECK(EmitBitwiseAndOrXor(f, ValType::I64, MIRType::Int64, + MWasmBinaryBitwise::SubOpcode::Xor)); + case uint16_t(Op::I64Shl): + CHECK(EmitShift(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64ShrS): + CHECK(EmitShift(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64ShrU): + CHECK(EmitUrsh(f, ValType::I64, MIRType::Int64)); + case uint16_t(Op::I64Rotl): + case uint16_t(Op::I64Rotr): + CHECK(EmitRotate(f, ValType::I64, Op(op.b0) == Op::I64Rotl)); + case uint16_t(Op::F32Abs): + CHECK(EmitUnaryWithType(f, ValType::F32, MIRType::Float32)); + case uint16_t(Op::F32Neg): + CHECK(EmitUnaryWithType(f, ValType::F32, MIRType::Float32)); + case uint16_t(Op::F32Ceil): + CHECK(EmitUnaryMathBuiltinCall(f, SASigCeilF)); + case uint16_t(Op::F32Floor): + CHECK(EmitUnaryMathBuiltinCall(f, SASigFloorF)); + case uint16_t(Op::F32Trunc): + CHECK(EmitUnaryMathBuiltinCall(f, SASigTruncF)); + case uint16_t(Op::F32Nearest): + CHECK(EmitUnaryMathBuiltinCall(f, SASigNearbyIntF)); + case uint16_t(Op::F32Sqrt): + CHECK(EmitUnaryWithType(f, ValType::F32, MIRType::Float32)); + case uint16_t(Op::F32Add): + CHECK(EmitAdd(f, ValType::F32, MIRType::Float32)); + case uint16_t(Op::F32Sub): + CHECK(EmitSub(f, ValType::F32, MIRType::Float32)); + case uint16_t(Op::F32Mul): + CHECK(EmitMul(f, ValType::F32, MIRType::Float32)); + case uint16_t(Op::F32Div): + CHECK(EmitDiv(f, ValType::F32, MIRType::Float32, + /* isUnsigned = */ false)); + case uint16_t(Op::F32Min): + case uint16_t(Op::F32Max): + CHECK(EmitMinMax(f, ValType::F32, MIRType::Float32, + Op(op.b0) == Op::F32Max)); + case uint16_t(Op::F32CopySign): + CHECK(EmitCopySign(f, ValType::F32)); + case uint16_t(Op::F64Abs): + CHECK(EmitUnaryWithType(f, ValType::F64, MIRType::Double)); + case uint16_t(Op::F64Neg): + CHECK(EmitUnaryWithType(f, ValType::F64, MIRType::Double)); + case uint16_t(Op::F64Ceil): + CHECK(EmitUnaryMathBuiltinCall(f, SASigCeilD)); + case uint16_t(Op::F64Floor): + CHECK(EmitUnaryMathBuiltinCall(f, SASigFloorD)); + case uint16_t(Op::F64Trunc): + CHECK(EmitUnaryMathBuiltinCall(f, SASigTruncD)); + case uint16_t(Op::F64Nearest): + CHECK(EmitUnaryMathBuiltinCall(f, SASigNearbyIntD)); + case uint16_t(Op::F64Sqrt): + CHECK(EmitUnaryWithType(f, ValType::F64, MIRType::Double)); + case uint16_t(Op::F64Add): + CHECK(EmitAdd(f, ValType::F64, MIRType::Double)); + case uint16_t(Op::F64Sub): + CHECK(EmitSub(f, ValType::F64, MIRType::Double)); + case uint16_t(Op::F64Mul): + CHECK(EmitMul(f, ValType::F64, MIRType::Double)); + case uint16_t(Op::F64Div): + CHECK(EmitDiv(f, ValType::F64, MIRType::Double, + /* isUnsigned = */ false)); + case uint16_t(Op::F64Min): + case uint16_t(Op::F64Max): + CHECK(EmitMinMax(f, ValType::F64, MIRType::Double, + Op(op.b0) == Op::F64Max)); + case uint16_t(Op::F64CopySign): + CHECK(EmitCopySign(f, ValType::F64)); + + // Conversions + case uint16_t(Op::I32WrapI64): + CHECK(EmitConversion(f, ValType::I64, ValType::I32)); + case uint16_t(Op::I32TruncF32S): + case uint16_t(Op::I32TruncF32U): + CHECK(EmitTruncate(f, ValType::F32, ValType::I32, + Op(op.b0) == Op::I32TruncF32U, false)); + case uint16_t(Op::I32TruncF64S): + case uint16_t(Op::I32TruncF64U): + CHECK(EmitTruncate(f, ValType::F64, ValType::I32, + Op(op.b0) == Op::I32TruncF64U, false)); + case uint16_t(Op::I64ExtendI32S): + case uint16_t(Op::I64ExtendI32U): + CHECK(EmitExtendI32(f, Op(op.b0) == Op::I64ExtendI32U)); + case uint16_t(Op::I64TruncF32S): + case uint16_t(Op::I64TruncF32U): + CHECK(EmitTruncate(f, ValType::F32, ValType::I64, + Op(op.b0) == Op::I64TruncF32U, false)); + case uint16_t(Op::I64TruncF64S): + case uint16_t(Op::I64TruncF64U): + CHECK(EmitTruncate(f, ValType::F64, ValType::I64, + Op(op.b0) == Op::I64TruncF64U, false)); + case uint16_t(Op::F32ConvertI32S): + CHECK(EmitConversion(f, ValType::I32, ValType::F32)); + case uint16_t(Op::F32ConvertI32U): + CHECK(EmitConversion(f, ValType::I32, + ValType::F32)); + case uint16_t(Op::F32ConvertI64S): + case uint16_t(Op::F32ConvertI64U): + CHECK(EmitConvertI64ToFloatingPoint(f, ValType::F32, MIRType::Float32, + Op(op.b0) == Op::F32ConvertI64U)); + case uint16_t(Op::F32DemoteF64): + CHECK(EmitConversion(f, ValType::F64, ValType::F32)); + case uint16_t(Op::F64ConvertI32S): + CHECK(EmitConversion(f, ValType::I32, ValType::F64)); + case uint16_t(Op::F64ConvertI32U): + CHECK(EmitConversion(f, ValType::I32, + ValType::F64)); + case uint16_t(Op::F64ConvertI64S): + case uint16_t(Op::F64ConvertI64U): + CHECK(EmitConvertI64ToFloatingPoint(f, ValType::F64, MIRType::Double, + Op(op.b0) == Op::F64ConvertI64U)); + case uint16_t(Op::F64PromoteF32): + CHECK(EmitConversion(f, ValType::F32, ValType::F64)); + + // Reinterpretations + case uint16_t(Op::I32ReinterpretF32): + CHECK(EmitReinterpret(f, ValType::I32, ValType::F32, MIRType::Int32)); + case uint16_t(Op::I64ReinterpretF64): + CHECK(EmitReinterpret(f, ValType::I64, ValType::F64, MIRType::Int64)); + case uint16_t(Op::F32ReinterpretI32): + CHECK(EmitReinterpret(f, ValType::F32, ValType::I32, MIRType::Float32)); + case uint16_t(Op::F64ReinterpretI64): + CHECK(EmitReinterpret(f, ValType::F64, ValType::I64, MIRType::Double)); + +#ifdef ENABLE_WASM_GC + case uint16_t(Op::RefEq): + if (!f.moduleEnv().gcEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitComparison(f, RefType::eq(), JSOp::Eq, + MCompare::Compare_RefOrNull)); +#endif + case uint16_t(Op::RefFunc): + CHECK(EmitRefFunc(f)); + case uint16_t(Op::RefNull): + CHECK(EmitRefNull(f)); + case uint16_t(Op::RefIsNull): + CHECK(EmitRefIsNull(f)); + + // Sign extensions + case uint16_t(Op::I32Extend8S): + CHECK(EmitSignExtend(f, 1, 4)); + case uint16_t(Op::I32Extend16S): + CHECK(EmitSignExtend(f, 2, 4)); + case uint16_t(Op::I64Extend8S): + CHECK(EmitSignExtend(f, 1, 8)); + case uint16_t(Op::I64Extend16S): + CHECK(EmitSignExtend(f, 2, 8)); + case uint16_t(Op::I64Extend32S): + CHECK(EmitSignExtend(f, 4, 8)); + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::RefAsNonNull): + if (!f.moduleEnv().functionReferencesEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitRefAsNonNull(f)); + case uint16_t(Op::BrOnNull): { + if (!f.moduleEnv().functionReferencesEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitBrOnNull(f)); + } + case uint16_t(Op::BrOnNonNull): { + if (!f.moduleEnv().functionReferencesEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitBrOnNonNull(f)); + } + case uint16_t(Op::CallRef): { + if (!f.moduleEnv().functionReferencesEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitCallRef(f)); + } +#endif + + // Gc operations +#ifdef ENABLE_WASM_GC + case uint16_t(Op::GcPrefix): { + if (!f.moduleEnv().gcEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(GcOp::StructNew): + CHECK(EmitStructNew(f)); + case uint32_t(GcOp::StructNewDefault): + CHECK(EmitStructNewDefault(f)); + case uint32_t(GcOp::StructSet): + CHECK(EmitStructSet(f)); + case uint32_t(GcOp::StructGet): + CHECK(EmitStructGet(f, FieldWideningOp::None)); + case uint32_t(GcOp::StructGetS): + CHECK(EmitStructGet(f, FieldWideningOp::Signed)); + case uint32_t(GcOp::StructGetU): + CHECK(EmitStructGet(f, FieldWideningOp::Unsigned)); + case uint32_t(GcOp::ArrayNew): + CHECK(EmitArrayNew(f)); + case uint32_t(GcOp::ArrayNewDefault): + CHECK(EmitArrayNewDefault(f)); + case uint32_t(GcOp::ArrayNewFixed): + CHECK(EmitArrayNewFixed(f)); + case uint32_t(GcOp::ArrayNewData): + CHECK(EmitArrayNewData(f)); + case uint32_t(GcOp::ArrayInitFromElemStaticV5): + case uint32_t(GcOp::ArrayNewElem): + CHECK(EmitArrayNewElem(f)); + case uint32_t(GcOp::ArraySet): + CHECK(EmitArraySet(f)); + case uint32_t(GcOp::ArrayGet): + CHECK(EmitArrayGet(f, FieldWideningOp::None)); + case uint32_t(GcOp::ArrayGetS): + CHECK(EmitArrayGet(f, FieldWideningOp::Signed)); + case uint32_t(GcOp::ArrayGetU): + CHECK(EmitArrayGet(f, FieldWideningOp::Unsigned)); + case uint32_t(GcOp::ArrayLenWithTypeIndex): + CHECK(EmitArrayLen(f, /*decodeIgnoredTypeIndex=*/true)); + case uint32_t(GcOp::ArrayLen): + CHECK(EmitArrayLen(f, /*decodeIgnoredTypeIndex=*/false)); + case uint32_t(GcOp::ArrayCopy): + CHECK(EmitArrayCopy(f)); + case uint32_t(GcOp::RefTestV5): + CHECK(EmitRefTestV5(f)); + case uint32_t(GcOp::RefCastV5): + CHECK(EmitRefCastV5(f)); + case uint32_t(GcOp::BrOnCast): + CHECK(EmitBrOnCast(f)); + case uint32_t(GcOp::BrOnCastV5): + CHECK(EmitBrOnCastCommonV5(f, /*onSuccess=*/true)); + case uint32_t(GcOp::BrOnCastFailV5): + CHECK(EmitBrOnCastCommonV5(f, /*onSuccess=*/false)); + case uint32_t(GcOp::BrOnCastHeapV5): + CHECK( + EmitBrOnCastHeapV5(f, /*onSuccess=*/true, /*nullable=*/false)); + case uint32_t(GcOp::BrOnCastHeapNullV5): + CHECK(EmitBrOnCastHeapV5(f, /*onSuccess=*/true, /*nullable=*/true)); + case uint32_t(GcOp::BrOnCastFailHeapV5): + CHECK( + EmitBrOnCastHeapV5(f, /*onSuccess=*/false, /*nullable=*/false)); + case uint32_t(GcOp::BrOnCastFailHeapNullV5): + CHECK( + EmitBrOnCastHeapV5(f, /*onSuccess=*/false, /*nullable=*/true)); + case uint32_t(GcOp::RefAsStructV5): + CHECK(EmitRefAsStructV5(f)); + case uint32_t(GcOp::BrOnNonStructV5): + CHECK(EmitBrOnNonStructV5(f)); + case uint32_t(GcOp::RefTest): + CHECK(EmitRefTest(f, /*nullable=*/false)); + case uint32_t(GcOp::RefTestNull): + CHECK(EmitRefTest(f, /*nullable=*/true)); + case uint32_t(GcOp::RefCast): + CHECK(EmitRefCast(f, /*nullable=*/false)); + case uint32_t(GcOp::RefCastNull): + CHECK(EmitRefCast(f, /*nullable=*/true)); + case uint16_t(GcOp::ExternInternalize): + CHECK(EmitExternInternalize(f)); + case uint16_t(GcOp::ExternExternalize): + CHECK(EmitExternExternalize(f)); + default: + return f.iter().unrecognizedOpcode(&op); + } // switch (op.b1) + break; + } +#endif + + // SIMD operations +#ifdef ENABLE_WASM_SIMD + case uint16_t(Op::SimdPrefix): { + if (!f.moduleEnv().simdAvailable()) { + return f.iter().unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(SimdOp::V128Const): + CHECK(EmitConstSimd128(f)); + case uint32_t(SimdOp::V128Load): + CHECK(EmitLoad(f, ValType::V128, Scalar::Simd128)); + case uint32_t(SimdOp::V128Store): + CHECK(EmitStore(f, ValType::V128, Scalar::Simd128)); + case uint32_t(SimdOp::V128And): + case uint32_t(SimdOp::V128Or): + case uint32_t(SimdOp::V128Xor): + case uint32_t(SimdOp::I8x16AvgrU): + case uint32_t(SimdOp::I16x8AvgrU): + case uint32_t(SimdOp::I8x16Add): + case uint32_t(SimdOp::I8x16AddSatS): + case uint32_t(SimdOp::I8x16AddSatU): + case uint32_t(SimdOp::I8x16MinS): + case uint32_t(SimdOp::I8x16MinU): + case uint32_t(SimdOp::I8x16MaxS): + case uint32_t(SimdOp::I8x16MaxU): + case uint32_t(SimdOp::I16x8Add): + case uint32_t(SimdOp::I16x8AddSatS): + case uint32_t(SimdOp::I16x8AddSatU): + case uint32_t(SimdOp::I16x8Mul): + case uint32_t(SimdOp::I16x8MinS): + case uint32_t(SimdOp::I16x8MinU): + case uint32_t(SimdOp::I16x8MaxS): + case uint32_t(SimdOp::I16x8MaxU): + case uint32_t(SimdOp::I32x4Add): + case uint32_t(SimdOp::I32x4Mul): + case uint32_t(SimdOp::I32x4MinS): + case uint32_t(SimdOp::I32x4MinU): + case uint32_t(SimdOp::I32x4MaxS): + case uint32_t(SimdOp::I32x4MaxU): + case uint32_t(SimdOp::I64x2Add): + case uint32_t(SimdOp::I64x2Mul): + case uint32_t(SimdOp::F32x4Add): + case uint32_t(SimdOp::F32x4Mul): + case uint32_t(SimdOp::F32x4Min): + case uint32_t(SimdOp::F32x4Max): + case uint32_t(SimdOp::F64x2Add): + case uint32_t(SimdOp::F64x2Mul): + case uint32_t(SimdOp::F64x2Min): + case uint32_t(SimdOp::F64x2Max): + case uint32_t(SimdOp::I8x16Eq): + case uint32_t(SimdOp::I8x16Ne): + case uint32_t(SimdOp::I16x8Eq): + case uint32_t(SimdOp::I16x8Ne): + case uint32_t(SimdOp::I32x4Eq): + case uint32_t(SimdOp::I32x4Ne): + case uint32_t(SimdOp::I64x2Eq): + case uint32_t(SimdOp::I64x2Ne): + case uint32_t(SimdOp::F32x4Eq): + case uint32_t(SimdOp::F32x4Ne): + case uint32_t(SimdOp::F64x2Eq): + case uint32_t(SimdOp::F64x2Ne): + case uint32_t(SimdOp::I32x4DotI16x8S): + case uint32_t(SimdOp::I16x8ExtmulLowI8x16S): + case uint32_t(SimdOp::I16x8ExtmulHighI8x16S): + case uint32_t(SimdOp::I16x8ExtmulLowI8x16U): + case uint32_t(SimdOp::I16x8ExtmulHighI8x16U): + case uint32_t(SimdOp::I32x4ExtmulLowI16x8S): + case uint32_t(SimdOp::I32x4ExtmulHighI16x8S): + case uint32_t(SimdOp::I32x4ExtmulLowI16x8U): + case uint32_t(SimdOp::I32x4ExtmulHighI16x8U): + case uint32_t(SimdOp::I64x2ExtmulLowI32x4S): + case uint32_t(SimdOp::I64x2ExtmulHighI32x4S): + case uint32_t(SimdOp::I64x2ExtmulLowI32x4U): + case uint32_t(SimdOp::I64x2ExtmulHighI32x4U): + case uint32_t(SimdOp::I16x8Q15MulrSatS): + CHECK(EmitBinarySimd128(f, /* commutative= */ true, SimdOp(op.b1))); + case uint32_t(SimdOp::V128AndNot): + case uint32_t(SimdOp::I8x16Sub): + case uint32_t(SimdOp::I8x16SubSatS): + case uint32_t(SimdOp::I8x16SubSatU): + case uint32_t(SimdOp::I16x8Sub): + case uint32_t(SimdOp::I16x8SubSatS): + case uint32_t(SimdOp::I16x8SubSatU): + case uint32_t(SimdOp::I32x4Sub): + case uint32_t(SimdOp::I64x2Sub): + case uint32_t(SimdOp::F32x4Sub): + case uint32_t(SimdOp::F32x4Div): + case uint32_t(SimdOp::F64x2Sub): + case uint32_t(SimdOp::F64x2Div): + case uint32_t(SimdOp::I8x16NarrowI16x8S): + case uint32_t(SimdOp::I8x16NarrowI16x8U): + case uint32_t(SimdOp::I16x8NarrowI32x4S): + case uint32_t(SimdOp::I16x8NarrowI32x4U): + case uint32_t(SimdOp::I8x16LtS): + case uint32_t(SimdOp::I8x16LtU): + case uint32_t(SimdOp::I8x16GtS): + case uint32_t(SimdOp::I8x16GtU): + case uint32_t(SimdOp::I8x16LeS): + case uint32_t(SimdOp::I8x16LeU): + case uint32_t(SimdOp::I8x16GeS): + case uint32_t(SimdOp::I8x16GeU): + case uint32_t(SimdOp::I16x8LtS): + case uint32_t(SimdOp::I16x8LtU): + case uint32_t(SimdOp::I16x8GtS): + case uint32_t(SimdOp::I16x8GtU): + case uint32_t(SimdOp::I16x8LeS): + case uint32_t(SimdOp::I16x8LeU): + case uint32_t(SimdOp::I16x8GeS): + case uint32_t(SimdOp::I16x8GeU): + case uint32_t(SimdOp::I32x4LtS): + case uint32_t(SimdOp::I32x4LtU): + case uint32_t(SimdOp::I32x4GtS): + case uint32_t(SimdOp::I32x4GtU): + case uint32_t(SimdOp::I32x4LeS): + case uint32_t(SimdOp::I32x4LeU): + case uint32_t(SimdOp::I32x4GeS): + case uint32_t(SimdOp::I32x4GeU): + case uint32_t(SimdOp::I64x2LtS): + case uint32_t(SimdOp::I64x2GtS): + case uint32_t(SimdOp::I64x2LeS): + case uint32_t(SimdOp::I64x2GeS): + case uint32_t(SimdOp::F32x4Lt): + case uint32_t(SimdOp::F32x4Gt): + case uint32_t(SimdOp::F32x4Le): + case uint32_t(SimdOp::F32x4Ge): + case uint32_t(SimdOp::F64x2Lt): + case uint32_t(SimdOp::F64x2Gt): + case uint32_t(SimdOp::F64x2Le): + case uint32_t(SimdOp::F64x2Ge): + case uint32_t(SimdOp::I8x16Swizzle): + case uint32_t(SimdOp::F32x4PMax): + case uint32_t(SimdOp::F32x4PMin): + case uint32_t(SimdOp::F64x2PMax): + case uint32_t(SimdOp::F64x2PMin): + CHECK( + EmitBinarySimd128(f, /* commutative= */ false, SimdOp(op.b1))); + case uint32_t(SimdOp::I8x16Splat): + case uint32_t(SimdOp::I16x8Splat): + case uint32_t(SimdOp::I32x4Splat): + CHECK(EmitSplatSimd128(f, ValType::I32, SimdOp(op.b1))); + case uint32_t(SimdOp::I64x2Splat): + CHECK(EmitSplatSimd128(f, ValType::I64, SimdOp(op.b1))); + case uint32_t(SimdOp::F32x4Splat): + CHECK(EmitSplatSimd128(f, ValType::F32, SimdOp(op.b1))); + case uint32_t(SimdOp::F64x2Splat): + CHECK(EmitSplatSimd128(f, ValType::F64, SimdOp(op.b1))); + case uint32_t(SimdOp::I8x16Neg): + case uint32_t(SimdOp::I16x8Neg): + case uint32_t(SimdOp::I16x8ExtendLowI8x16S): + case uint32_t(SimdOp::I16x8ExtendHighI8x16S): + case uint32_t(SimdOp::I16x8ExtendLowI8x16U): + case uint32_t(SimdOp::I16x8ExtendHighI8x16U): + case uint32_t(SimdOp::I32x4Neg): + case uint32_t(SimdOp::I32x4ExtendLowI16x8S): + case uint32_t(SimdOp::I32x4ExtendHighI16x8S): + case uint32_t(SimdOp::I32x4ExtendLowI16x8U): + case uint32_t(SimdOp::I32x4ExtendHighI16x8U): + case uint32_t(SimdOp::I32x4TruncSatF32x4S): + case uint32_t(SimdOp::I32x4TruncSatF32x4U): + case uint32_t(SimdOp::I64x2Neg): + case uint32_t(SimdOp::I64x2ExtendLowI32x4S): + case uint32_t(SimdOp::I64x2ExtendHighI32x4S): + case uint32_t(SimdOp::I64x2ExtendLowI32x4U): + case uint32_t(SimdOp::I64x2ExtendHighI32x4U): + case uint32_t(SimdOp::F32x4Abs): + case uint32_t(SimdOp::F32x4Neg): + case uint32_t(SimdOp::F32x4Sqrt): + case uint32_t(SimdOp::F32x4ConvertI32x4S): + case uint32_t(SimdOp::F32x4ConvertI32x4U): + case uint32_t(SimdOp::F64x2Abs): + case uint32_t(SimdOp::F64x2Neg): + case uint32_t(SimdOp::F64x2Sqrt): + case uint32_t(SimdOp::V128Not): + case uint32_t(SimdOp::I8x16Popcnt): + case uint32_t(SimdOp::I8x16Abs): + case uint32_t(SimdOp::I16x8Abs): + case uint32_t(SimdOp::I32x4Abs): + case uint32_t(SimdOp::I64x2Abs): + case uint32_t(SimdOp::F32x4Ceil): + case uint32_t(SimdOp::F32x4Floor): + case uint32_t(SimdOp::F32x4Trunc): + case uint32_t(SimdOp::F32x4Nearest): + case uint32_t(SimdOp::F64x2Ceil): + case uint32_t(SimdOp::F64x2Floor): + case uint32_t(SimdOp::F64x2Trunc): + case uint32_t(SimdOp::F64x2Nearest): + case uint32_t(SimdOp::F32x4DemoteF64x2Zero): + case uint32_t(SimdOp::F64x2PromoteLowF32x4): + case uint32_t(SimdOp::F64x2ConvertLowI32x4S): + case uint32_t(SimdOp::F64x2ConvertLowI32x4U): + case uint32_t(SimdOp::I32x4TruncSatF64x2SZero): + case uint32_t(SimdOp::I32x4TruncSatF64x2UZero): + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S): + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U): + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S): + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U): + CHECK(EmitUnarySimd128(f, SimdOp(op.b1))); + case uint32_t(SimdOp::V128AnyTrue): + case uint32_t(SimdOp::I8x16AllTrue): + case uint32_t(SimdOp::I16x8AllTrue): + case uint32_t(SimdOp::I32x4AllTrue): + case uint32_t(SimdOp::I64x2AllTrue): + case uint32_t(SimdOp::I8x16Bitmask): + case uint32_t(SimdOp::I16x8Bitmask): + case uint32_t(SimdOp::I32x4Bitmask): + case uint32_t(SimdOp::I64x2Bitmask): + CHECK(EmitReduceSimd128(f, SimdOp(op.b1))); + case uint32_t(SimdOp::I8x16Shl): + case uint32_t(SimdOp::I8x16ShrS): + case uint32_t(SimdOp::I8x16ShrU): + case uint32_t(SimdOp::I16x8Shl): + case uint32_t(SimdOp::I16x8ShrS): + case uint32_t(SimdOp::I16x8ShrU): + case uint32_t(SimdOp::I32x4Shl): + case uint32_t(SimdOp::I32x4ShrS): + case uint32_t(SimdOp::I32x4ShrU): + case uint32_t(SimdOp::I64x2Shl): + case uint32_t(SimdOp::I64x2ShrS): + case uint32_t(SimdOp::I64x2ShrU): + CHECK(EmitShiftSimd128(f, SimdOp(op.b1))); + case uint32_t(SimdOp::I8x16ExtractLaneS): + case uint32_t(SimdOp::I8x16ExtractLaneU): + CHECK(EmitExtractLaneSimd128(f, ValType::I32, 16, SimdOp(op.b1))); + case uint32_t(SimdOp::I16x8ExtractLaneS): + case uint32_t(SimdOp::I16x8ExtractLaneU): + CHECK(EmitExtractLaneSimd128(f, ValType::I32, 8, SimdOp(op.b1))); + case uint32_t(SimdOp::I32x4ExtractLane): + CHECK(EmitExtractLaneSimd128(f, ValType::I32, 4, SimdOp(op.b1))); + case uint32_t(SimdOp::I64x2ExtractLane): + CHECK(EmitExtractLaneSimd128(f, ValType::I64, 2, SimdOp(op.b1))); + case uint32_t(SimdOp::F32x4ExtractLane): + CHECK(EmitExtractLaneSimd128(f, ValType::F32, 4, SimdOp(op.b1))); + case uint32_t(SimdOp::F64x2ExtractLane): + CHECK(EmitExtractLaneSimd128(f, ValType::F64, 2, SimdOp(op.b1))); + case uint32_t(SimdOp::I8x16ReplaceLane): + CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 16, SimdOp(op.b1))); + case uint32_t(SimdOp::I16x8ReplaceLane): + CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 8, SimdOp(op.b1))); + case uint32_t(SimdOp::I32x4ReplaceLane): + CHECK(EmitReplaceLaneSimd128(f, ValType::I32, 4, SimdOp(op.b1))); + case uint32_t(SimdOp::I64x2ReplaceLane): + CHECK(EmitReplaceLaneSimd128(f, ValType::I64, 2, SimdOp(op.b1))); + case uint32_t(SimdOp::F32x4ReplaceLane): + CHECK(EmitReplaceLaneSimd128(f, ValType::F32, 4, SimdOp(op.b1))); + case uint32_t(SimdOp::F64x2ReplaceLane): + CHECK(EmitReplaceLaneSimd128(f, ValType::F64, 2, SimdOp(op.b1))); + case uint32_t(SimdOp::V128Bitselect): + CHECK(EmitTernarySimd128(f, SimdOp(op.b1))); + case uint32_t(SimdOp::I8x16Shuffle): + CHECK(EmitShuffleSimd128(f)); + case uint32_t(SimdOp::V128Load8Splat): + CHECK(EmitLoadSplatSimd128(f, Scalar::Uint8, SimdOp::I8x16Splat)); + case uint32_t(SimdOp::V128Load16Splat): + CHECK(EmitLoadSplatSimd128(f, Scalar::Uint16, SimdOp::I16x8Splat)); + case uint32_t(SimdOp::V128Load32Splat): + CHECK(EmitLoadSplatSimd128(f, Scalar::Float32, SimdOp::I32x4Splat)); + case uint32_t(SimdOp::V128Load64Splat): + CHECK(EmitLoadSplatSimd128(f, Scalar::Float64, SimdOp::I64x2Splat)); + case uint32_t(SimdOp::V128Load8x8S): + case uint32_t(SimdOp::V128Load8x8U): + case uint32_t(SimdOp::V128Load16x4S): + case uint32_t(SimdOp::V128Load16x4U): + case uint32_t(SimdOp::V128Load32x2S): + case uint32_t(SimdOp::V128Load32x2U): + CHECK(EmitLoadExtendSimd128(f, SimdOp(op.b1))); + case uint32_t(SimdOp::V128Load32Zero): + CHECK(EmitLoadZeroSimd128(f, Scalar::Float32, 4)); + case uint32_t(SimdOp::V128Load64Zero): + CHECK(EmitLoadZeroSimd128(f, Scalar::Float64, 8)); + case uint32_t(SimdOp::V128Load8Lane): + CHECK(EmitLoadLaneSimd128(f, 1)); + case uint32_t(SimdOp::V128Load16Lane): + CHECK(EmitLoadLaneSimd128(f, 2)); + case uint32_t(SimdOp::V128Load32Lane): + CHECK(EmitLoadLaneSimd128(f, 4)); + case uint32_t(SimdOp::V128Load64Lane): + CHECK(EmitLoadLaneSimd128(f, 8)); + case uint32_t(SimdOp::V128Store8Lane): + CHECK(EmitStoreLaneSimd128(f, 1)); + case uint32_t(SimdOp::V128Store16Lane): + CHECK(EmitStoreLaneSimd128(f, 2)); + case uint32_t(SimdOp::V128Store32Lane): + CHECK(EmitStoreLaneSimd128(f, 4)); + case uint32_t(SimdOp::V128Store64Lane): + CHECK(EmitStoreLaneSimd128(f, 8)); +# ifdef ENABLE_WASM_RELAXED_SIMD + case uint32_t(SimdOp::F32x4RelaxedFma): + case uint32_t(SimdOp::F32x4RelaxedFnma): + case uint32_t(SimdOp::F64x2RelaxedFma): + case uint32_t(SimdOp::F64x2RelaxedFnma): + case uint32_t(SimdOp::I8x16RelaxedLaneSelect): + case uint32_t(SimdOp::I16x8RelaxedLaneSelect): + case uint32_t(SimdOp::I32x4RelaxedLaneSelect): + case uint32_t(SimdOp::I64x2RelaxedLaneSelect): + case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS): { + if (!f.moduleEnv().v128RelaxedEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitTernarySimd128(f, SimdOp(op.b1))); + } + case uint32_t(SimdOp::F32x4RelaxedMin): + case uint32_t(SimdOp::F32x4RelaxedMax): + case uint32_t(SimdOp::F64x2RelaxedMin): + case uint32_t(SimdOp::F64x2RelaxedMax): + case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): { + if (!f.moduleEnv().v128RelaxedEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitBinarySimd128(f, /* commutative= */ true, SimdOp(op.b1))); + } + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S): + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U): + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero): + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): { + if (!f.moduleEnv().v128RelaxedEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitUnarySimd128(f, SimdOp(op.b1))); + } + case uint32_t(SimdOp::I8x16RelaxedSwizzle): + case uint32_t(SimdOp::I16x8DotI8x16I7x16S): { + if (!f.moduleEnv().v128RelaxedEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK( + EmitBinarySimd128(f, /* commutative= */ false, SimdOp(op.b1))); + } +# endif + + default: + return f.iter().unrecognizedOpcode(&op); + } // switch (op.b1) + break; + } +#endif + + // Miscellaneous operations + case uint16_t(Op::MiscPrefix): { + switch (op.b1) { + case uint32_t(MiscOp::I32TruncSatF32S): + case uint32_t(MiscOp::I32TruncSatF32U): + CHECK(EmitTruncate(f, ValType::F32, ValType::I32, + MiscOp(op.b1) == MiscOp::I32TruncSatF32U, true)); + case uint32_t(MiscOp::I32TruncSatF64S): + case uint32_t(MiscOp::I32TruncSatF64U): + CHECK(EmitTruncate(f, ValType::F64, ValType::I32, + MiscOp(op.b1) == MiscOp::I32TruncSatF64U, true)); + case uint32_t(MiscOp::I64TruncSatF32S): + case uint32_t(MiscOp::I64TruncSatF32U): + CHECK(EmitTruncate(f, ValType::F32, ValType::I64, + MiscOp(op.b1) == MiscOp::I64TruncSatF32U, true)); + case uint32_t(MiscOp::I64TruncSatF64S): + case uint32_t(MiscOp::I64TruncSatF64U): + CHECK(EmitTruncate(f, ValType::F64, ValType::I64, + MiscOp(op.b1) == MiscOp::I64TruncSatF64U, true)); + case uint32_t(MiscOp::MemoryCopy): + CHECK(EmitMemCopy(f)); + case uint32_t(MiscOp::DataDrop): + CHECK(EmitDataOrElemDrop(f, /*isData=*/true)); + case uint32_t(MiscOp::MemoryFill): + CHECK(EmitMemFill(f)); + case uint32_t(MiscOp::MemoryInit): + CHECK(EmitMemOrTableInit(f, /*isMem=*/true)); + case uint32_t(MiscOp::TableCopy): + CHECK(EmitTableCopy(f)); + case uint32_t(MiscOp::ElemDrop): + CHECK(EmitDataOrElemDrop(f, /*isData=*/false)); + case uint32_t(MiscOp::TableInit): + CHECK(EmitMemOrTableInit(f, /*isMem=*/false)); + case uint32_t(MiscOp::TableFill): + CHECK(EmitTableFill(f)); +#if ENABLE_WASM_MEMORY_CONTROL + case uint32_t(MiscOp::MemoryDiscard): { + if (!f.moduleEnv().memoryControlEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitMemDiscard(f)); + } +#endif + case uint32_t(MiscOp::TableGrow): + CHECK(EmitTableGrow(f)); + case uint32_t(MiscOp::TableSize): + CHECK(EmitTableSize(f)); + default: + return f.iter().unrecognizedOpcode(&op); + } + break; + } + + // Thread operations + case uint16_t(Op::ThreadPrefix): { + // Though thread ops can be used on nonshared memories, we make them + // unavailable if shared memory has been disabled in the prefs, for + // maximum predictability and safety and consistency with JS. + if (f.moduleEnv().sharedMemoryEnabled() == Shareable::False) { + return f.iter().unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(ThreadOp::Wake): + CHECK(EmitWake(f)); + + case uint32_t(ThreadOp::I32Wait): + CHECK(EmitWait(f, ValType::I32, 4)); + case uint32_t(ThreadOp::I64Wait): + CHECK(EmitWait(f, ValType::I64, 8)); + case uint32_t(ThreadOp::Fence): + CHECK(EmitFence(f)); + + case uint32_t(ThreadOp::I32AtomicLoad): + CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicLoad): + CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicLoad8U): + CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicLoad16U): + CHECK(EmitAtomicLoad(f, ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicLoad8U): + CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicLoad16U): + CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicLoad32U): + CHECK(EmitAtomicLoad(f, ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicStore): + CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicStore): + CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicStore8U): + CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicStore16U): + CHECK(EmitAtomicStore(f, ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicStore8U): + CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicStore16U): + CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicStore32U): + CHECK(EmitAtomicStore(f, ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicAdd): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, + AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, + AtomicFetchAddOp)); + case uint32_t(ThreadOp::I32AtomicAdd8U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, + AtomicFetchAddOp)); + case uint32_t(ThreadOp::I32AtomicAdd16U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, + AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd8U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, + AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd16U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, + AtomicFetchAddOp)); + case uint32_t(ThreadOp::I64AtomicAdd32U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, + AtomicFetchAddOp)); + + case uint32_t(ThreadOp::I32AtomicSub): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, + AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, + AtomicFetchSubOp)); + case uint32_t(ThreadOp::I32AtomicSub8U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, + AtomicFetchSubOp)); + case uint32_t(ThreadOp::I32AtomicSub16U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, + AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub8U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, + AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub16U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, + AtomicFetchSubOp)); + case uint32_t(ThreadOp::I64AtomicSub32U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, + AtomicFetchSubOp)); + + case uint32_t(ThreadOp::I32AtomicAnd): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, + AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, + AtomicFetchAndOp)); + case uint32_t(ThreadOp::I32AtomicAnd8U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, + AtomicFetchAndOp)); + case uint32_t(ThreadOp::I32AtomicAnd16U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, + AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd8U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, + AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd16U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, + AtomicFetchAndOp)); + case uint32_t(ThreadOp::I64AtomicAnd32U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, + AtomicFetchAndOp)); + + case uint32_t(ThreadOp::I32AtomicOr): + CHECK( + EmitAtomicRMW(f, ValType::I32, Scalar::Int32, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr): + CHECK( + EmitAtomicRMW(f, ValType::I64, Scalar::Int64, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I32AtomicOr8U): + CHECK( + EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I32AtomicOr16U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, + AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr8U): + CHECK( + EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr16U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, + AtomicFetchOrOp)); + case uint32_t(ThreadOp::I64AtomicOr32U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, + AtomicFetchOrOp)); + + case uint32_t(ThreadOp::I32AtomicXor): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Int32, + AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Int64, + AtomicFetchXorOp)); + case uint32_t(ThreadOp::I32AtomicXor8U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint8, + AtomicFetchXorOp)); + case uint32_t(ThreadOp::I32AtomicXor16U): + CHECK(EmitAtomicRMW(f, ValType::I32, Scalar::Uint16, + AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor8U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint8, + AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor16U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint16, + AtomicFetchXorOp)); + case uint32_t(ThreadOp::I64AtomicXor32U): + CHECK(EmitAtomicRMW(f, ValType::I64, Scalar::Uint32, + AtomicFetchXorOp)); + + case uint32_t(ThreadOp::I32AtomicXchg): + CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicXchg): + CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicXchg8U): + CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicXchg16U): + CHECK(EmitAtomicXchg(f, ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicXchg8U): + CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicXchg16U): + CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicXchg32U): + CHECK(EmitAtomicXchg(f, ValType::I64, Scalar::Uint32)); + + case uint32_t(ThreadOp::I32AtomicCmpXchg): + CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Int32)); + case uint32_t(ThreadOp::I64AtomicCmpXchg): + CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Int64)); + case uint32_t(ThreadOp::I32AtomicCmpXchg8U): + CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Uint8)); + case uint32_t(ThreadOp::I32AtomicCmpXchg16U): + CHECK(EmitAtomicCmpXchg(f, ValType::I32, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicCmpXchg8U): + CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint8)); + case uint32_t(ThreadOp::I64AtomicCmpXchg16U): + CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint16)); + case uint32_t(ThreadOp::I64AtomicCmpXchg32U): + CHECK(EmitAtomicCmpXchg(f, ValType::I64, Scalar::Uint32)); + + default: + return f.iter().unrecognizedOpcode(&op); + } + break; + } + + // asm.js-specific operators + case uint16_t(Op::MozPrefix): { + if (op.b1 == uint32_t(MozOp::Intrinsic)) { + if (!f.moduleEnv().intrinsicsEnabled()) { + return f.iter().unrecognizedOpcode(&op); + } + CHECK(EmitIntrinsic(f)); + } + + if (!f.moduleEnv().isAsmJS()) { + return f.iter().unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(MozOp::TeeGlobal): + CHECK(EmitTeeGlobal(f)); + case uint32_t(MozOp::I32Min): + case uint32_t(MozOp::I32Max): + CHECK(EmitMinMax(f, ValType::I32, MIRType::Int32, + MozOp(op.b1) == MozOp::I32Max)); + case uint32_t(MozOp::I32Neg): + CHECK(EmitUnaryWithType(f, ValType::I32, MIRType::Int32)); + case uint32_t(MozOp::I32BitNot): + CHECK(EmitBitNot(f, ValType::I32)); + case uint32_t(MozOp::I32Abs): + CHECK(EmitUnaryWithType(f, ValType::I32, MIRType::Int32)); + case uint32_t(MozOp::F32TeeStoreF64): + CHECK(EmitTeeStoreWithCoercion(f, ValType::F32, Scalar::Float64)); + case uint32_t(MozOp::F64TeeStoreF32): + CHECK(EmitTeeStoreWithCoercion(f, ValType::F64, Scalar::Float32)); + case uint32_t(MozOp::I32TeeStore8): + CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int8)); + case uint32_t(MozOp::I32TeeStore16): + CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int16)); + case uint32_t(MozOp::I64TeeStore8): + CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int8)); + case uint32_t(MozOp::I64TeeStore16): + CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int16)); + case uint32_t(MozOp::I64TeeStore32): + CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int32)); + case uint32_t(MozOp::I32TeeStore): + CHECK(EmitTeeStore(f, ValType::I32, Scalar::Int32)); + case uint32_t(MozOp::I64TeeStore): + CHECK(EmitTeeStore(f, ValType::I64, Scalar::Int64)); + case uint32_t(MozOp::F32TeeStore): + CHECK(EmitTeeStore(f, ValType::F32, Scalar::Float32)); + case uint32_t(MozOp::F64TeeStore): + CHECK(EmitTeeStore(f, ValType::F64, Scalar::Float64)); + case uint32_t(MozOp::F64Mod): + CHECK(EmitRem(f, ValType::F64, MIRType::Double, + /* isUnsigned = */ false)); + case uint32_t(MozOp::F64SinNative): + CHECK(EmitUnaryMathBuiltinCall(f, SASigSinNativeD)); + case uint32_t(MozOp::F64SinFdlibm): + CHECK(EmitUnaryMathBuiltinCall(f, SASigSinFdlibmD)); + case uint32_t(MozOp::F64CosNative): + CHECK(EmitUnaryMathBuiltinCall(f, SASigCosNativeD)); + case uint32_t(MozOp::F64CosFdlibm): + CHECK(EmitUnaryMathBuiltinCall(f, SASigCosFdlibmD)); + case uint32_t(MozOp::F64TanNative): + CHECK(EmitUnaryMathBuiltinCall(f, SASigTanNativeD)); + case uint32_t(MozOp::F64TanFdlibm): + CHECK(EmitUnaryMathBuiltinCall(f, SASigTanFdlibmD)); + case uint32_t(MozOp::F64Asin): + CHECK(EmitUnaryMathBuiltinCall(f, SASigASinD)); + case uint32_t(MozOp::F64Acos): + CHECK(EmitUnaryMathBuiltinCall(f, SASigACosD)); + case uint32_t(MozOp::F64Atan): + CHECK(EmitUnaryMathBuiltinCall(f, SASigATanD)); + case uint32_t(MozOp::F64Exp): + CHECK(EmitUnaryMathBuiltinCall(f, SASigExpD)); + case uint32_t(MozOp::F64Log): + CHECK(EmitUnaryMathBuiltinCall(f, SASigLogD)); + case uint32_t(MozOp::F64Pow): + CHECK(EmitBinaryMathBuiltinCall(f, SASigPowD)); + case uint32_t(MozOp::F64Atan2): + CHECK(EmitBinaryMathBuiltinCall(f, SASigATan2D)); + case uint32_t(MozOp::OldCallDirect): + CHECK(EmitCall(f, /* asmJSFuncDef = */ true)); + case uint32_t(MozOp::OldCallIndirect): + CHECK(EmitCallIndirect(f, /* oldStyle = */ true)); + + default: + return f.iter().unrecognizedOpcode(&op); + } + break; + } + + default: + return f.iter().unrecognizedOpcode(&op); + } + } + + MOZ_CRASH("unreachable"); + +#undef CHECK +} + +bool wasm::IonCompileFunctions(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + LifoAlloc& lifo, + const FuncCompileInputVector& inputs, + CompiledCode* code, UniqueChars* error) { + MOZ_ASSERT(compilerEnv.tier() == Tier::Optimized); + MOZ_ASSERT(compilerEnv.debug() == DebugEnabled::False); + + TempAllocator alloc(&lifo); + JitContext jitContext; + MOZ_ASSERT(IsCompilingWasm()); + WasmMacroAssembler masm(alloc, moduleEnv); +#if defined(JS_CODEGEN_ARM64) + masm.SetStackPointer64(PseudoStackPointer64); +#endif + + // Swap in already-allocated empty vectors to avoid malloc/free. + MOZ_ASSERT(code->empty()); + if (!code->swap(masm)) { + return false; + } + + // Create a description of the stack layout created by GenerateTrapExit(). + RegisterOffsets trapExitLayout; + size_t trapExitLayoutNumWords; + GenerateTrapExitRegisterOffsets(&trapExitLayout, &trapExitLayoutNumWords); + + for (const FuncCompileInput& func : inputs) { + JitSpewCont(JitSpew_Codegen, "\n"); + JitSpew(JitSpew_Codegen, + "# ================================" + "=================================="); + JitSpew(JitSpew_Codegen, "# =="); + JitSpew(JitSpew_Codegen, + "# wasm::IonCompileFunctions: starting on function index %d", + (int)func.index); + + Decoder d(func.begin, func.end, func.lineOrBytecode, error); + + // Build the local types vector. + + const FuncType& funcType = *moduleEnv.funcs[func.index].type; + ValTypeVector locals; + if (!locals.appendAll(funcType.args())) { + return false; + } + if (!DecodeLocalEntries(d, *moduleEnv.types, moduleEnv.features, &locals)) { + return false; + } + + // Set up for Ion compilation. + + const JitCompileOptions options; + MIRGraph graph(&alloc); + CompileInfo compileInfo(locals.length()); + MIRGenerator mir(nullptr, options, &alloc, &graph, &compileInfo, + IonOptimizations.get(OptimizationLevel::Wasm)); + if (moduleEnv.usesMemory()) { + if (moduleEnv.memory->indexType() == IndexType::I32) { + mir.initMinWasmHeapLength(moduleEnv.memory->initialLength32()); + } else { + mir.initMinWasmHeapLength(moduleEnv.memory->initialLength64()); + } + } + + // Build MIR graph + { + FunctionCompiler f(moduleEnv, d, func, locals, mir, masm.tryNotes()); + if (!f.init()) { + return false; + } + + if (!f.startBlock()) { + return false; + } + + if (!EmitBodyExprs(f)) { + return false; + } + + f.finish(); + } + + // Compile MIR graph + { + jit::SpewBeginWasmFunction(&mir, func.index); + jit::AutoSpewEndFunction spewEndFunction(&mir); + + if (!OptimizeMIR(&mir)) { + return false; + } + + LIRGraph* lir = GenerateLIR(&mir); + if (!lir) { + return false; + } + + CodeGenerator codegen(&mir, lir, &masm); + + BytecodeOffset prologueTrapOffset(func.lineOrBytecode); + FuncOffsets offsets; + ArgTypeVector args(funcType); + if (!codegen.generateWasm(CallIndirectId::forFunc(moduleEnv, func.index), + prologueTrapOffset, args, trapExitLayout, + trapExitLayoutNumWords, &offsets, + &code->stackMaps, &d)) { + return false; + } + + if (!code->codeRanges.emplaceBack(func.index, func.lineOrBytecode, + offsets)) { + return false; + } + } + + JitSpew(JitSpew_Codegen, + "# wasm::IonCompileFunctions: completed function index %d", + (int)func.index); + JitSpew(JitSpew_Codegen, "# =="); + JitSpew(JitSpew_Codegen, + "# ================================" + "=================================="); + JitSpewCont(JitSpew_Codegen, "\n"); + } + + masm.finish(); + if (masm.oom()) { + return false; + } + + return code->swap(masm); +} + +bool js::wasm::IonPlatformSupport() { +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ + defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) + return true; +#else + return false; +#endif +} diff --git a/js/src/wasm/WasmIonCompile.h b/js/src/wasm/WasmIonCompile.h new file mode 100644 index 0000000000..f583cbad1f --- /dev/null +++ b/js/src/wasm/WasmIonCompile.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_ion_compile_h +#define wasm_ion_compile_h + +#include "wasm/WasmGenerator.h" + +namespace js { +namespace wasm { + +// Return whether IonCompileFunction() can generate code on the current device. +// Usually you do *not* want this, you want IonAvailable(). +[[nodiscard]] bool IonPlatformSupport(); + +// Generates very fast code at the expense of compilation time. +[[nodiscard]] bool IonCompileFunctions(const ModuleEnvironment& moduleEnv, + const CompilerEnvironment& compilerEnv, + LifoAlloc& lifo, + const FuncCompileInputVector& inputs, + CompiledCode* code, UniqueChars* error); + +} // namespace wasm +} // namespace js + +#endif // wasm_ion_compile_h diff --git a/js/src/wasm/WasmJS.cpp b/js/src/wasm/WasmJS.cpp new file mode 100644 index 0000000000..75a37dc194 --- /dev/null +++ b/js/src/wasm/WasmJS.cpp @@ -0,0 +1,5524 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmJS.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/EndianUtils.h" +#include "mozilla/Maybe.h" +#include "mozilla/RangedPtr.h" + +#include +#include + +#include "jsapi.h" +#include "jsexn.h" + +#include "ds/IdValuePair.h" // js::IdValuePair +#include "frontend/FrontendContext.h" // AutoReportFrontendContext +#include "gc/GCContext.h" +#include "jit/AtomicOperations.h" +#include "jit/FlushICache.h" +#include "jit/JitContext.h" +#include "jit/JitOptions.h" +#include "jit/Simulator.h" +#include "js/ForOfIterator.h" +#include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_* +#include "js/Printf.h" +#include "js/PropertyAndElement.h" // JS_DefineProperty, JS_GetProperty +#include "js/PropertySpec.h" // JS_{PS,FN}{,_END} +#include "js/Stack.h" // BuildStackString +#include "js/StreamConsumer.h" +#include "util/StringBuffer.h" +#include "util/Text.h" +#include "vm/ErrorObject.h" +#include "vm/FunctionFlags.h" // js::FunctionFlags +#include "vm/GlobalObject.h" // js::GlobalObject +#include "vm/HelperThreadState.h" // js::PromiseHelperTask +#include "vm/Interpreter.h" +#include "vm/JSFunction.h" +#include "vm/PlainObject.h" // js::PlainObject +#include "vm/PromiseObject.h" // js::PromiseObject +#include "vm/SharedArrayObject.h" +#include "vm/StringType.h" +#include "vm/Warnings.h" // js::WarnNumberASCII +#include "vm/WellKnownAtom.h" // js_*_str +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCompile.h" +#include "wasm/WasmDebug.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmIntrinsic.h" +#include "wasm/WasmIonCompile.h" +#include "wasm/WasmMemory.h" +#include "wasm/WasmModule.h" +#include "wasm/WasmProcess.h" +#include "wasm/WasmSignalHandlers.h" +#include "wasm/WasmStubs.h" +#include "wasm/WasmValidate.h" + +#include "gc/GCContext-inl.h" +#include "gc/StableCellHasher-inl.h" +#include "vm/ArrayBufferObject-inl.h" +#include "vm/JSObject-inl.h" +#include "vm/NativeObject-inl.h" +#include "wasm/WasmInstance-inl.h" + +/* + * [SMDOC] WebAssembly code rules (evolving) + * + * TlsContext.get() is only to be invoked from functions that have been invoked + * _directly_ by generated code as cold(!) Builtin calls, from code that is + * only used by signal handlers, or from helper functions that have been + * called _directly_ from a simulator. All other code shall pass in a + * JSContext* to functions that need it, or an Instance* or Instance* since + * the context is available through them. + * + * Code that uses TlsContext.get() shall annotate each such call with the + * reason why the call is OK. + */ + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::CheckedInt; +using mozilla::Nothing; +using mozilla::RangedPtr; +using mozilla::Span; + +// About the fuzzer intercession points: If fuzzing has been selected and only a +// single compiler has been selected then we will disable features that are not +// supported by that single compiler. This is strictly a concession to the +// fuzzer infrastructure. + +static inline bool IsFuzzingIon(JSContext* cx) { + return IsFuzzing() && !cx->options().wasmBaseline() && + cx->options().wasmIon(); +} + +// These functions read flags and apply fuzzing intercession policies. Never go +// directly to the flags in code below, always go via these accessors. + +static inline bool WasmThreadsFlag(JSContext* cx) { + return cx->realm() && + cx->realm()->creationOptions().getSharedMemoryAndAtomicsEnabled(); +} + +#define WASM_FEATURE(NAME, LOWER_NAME, COMPILE_PRED, COMPILER_PRED, FLAG_PRED, \ + ...) \ + static inline bool Wasm##NAME##Flag(JSContext* cx) { \ + return (COMPILE_PRED) && (FLAG_PRED) && cx->options().wasm##NAME(); \ + } +JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE); +#undef WASM_FEATURE + +static inline bool WasmDebuggerActive(JSContext* cx) { + if (IsFuzzingIon(cx)) { + return false; + } + return cx->realm() && cx->realm()->debuggerObservesWasm(); +} + +/* + * [SMDOC] Compiler and feature selection; compiler and feature availability. + * + * In order to make the computation of whether a wasm feature or wasm compiler + * is available predictable, we have established some rules, and implemented + * those rules. + * + * Code elsewhere should use the predicates below to test for features and + * compilers, it should never try to compute feature and compiler availability + * in other ways. + * + * At the outset, there is a set of selected compilers C containing at most one + * baseline compiler [*] and at most one optimizing compiler [**], and a set of + * selected features F. These selections come from defaults and from overrides + * by command line switches in the shell and javascript.option.wasm_X in the + * browser. Defaults for both features and compilers may be platform specific, + * for example, some compilers may not be available on some platforms because + * they do not support the architecture at all or they do not support features + * that must be enabled by default on the platform. + * + * [*] Currently we have only one, "baseline" aka "Rabaldr", but other + * implementations have additional baseline translators, eg from wasm + * bytecode to an internal code processed by an interpreter. + * + * [**] Currently we have only one, "ion" aka "Baldr". + * + * + * Compiler availability: + * + * The set of features F induces a set of available compilers A: these are the + * compilers that all support all the features in F. (Some of these compilers + * may not be in the set C.) + * + * The sets C and A are intersected, yielding a set of enabled compilers E. + * Notably, the set E may be empty, in which case wasm is effectively disabled + * (though the WebAssembly object is still present in the global environment). + * + * An important consequence is that selecting a feature that is not supported by + * a particular compiler disables that compiler completely -- there is no notion + * of a compiler being available but suddenly failing when an unsupported + * feature is used by a program. If a compiler is available, it supports all + * the features that have been selected. + * + * Equally important, a feature cannot be enabled by default on a platform if + * the feature is not supported by all the compilers we wish to have enabled by + * default on the platform. We MUST by-default disable features on a platform + * that are not supported by all the compilers on the platform. + * + * In a shell build, the testing functions wasmCompilersPresent, + * wasmCompileMode, and wasmIonDisabledByFeatures can be used to probe compiler + * availability and the reasons for a compiler being unavailable. + * + * + * Feature availability: + * + * A feature is available if it is selected and there is at least one available + * compiler that implements it. + * + * For example, --wasm-gc selects the GC feature, and if Baseline is available + * then the feature is available. + * + * In a shell build, there are per-feature testing functions (of the form + * wasmFeatureEnabled) to probe whether specific features are available. + */ + +// Compiler availability predicates. These must be kept in sync with the +// feature predicates in the next section below. +// +// These can't call the feature predicates since the feature predicates call +// back to these predicates. So there will be a small amount of duplicated +// logic here, but as compilers reach feature parity that duplication will go +// away. + +bool wasm::BaselineAvailable(JSContext* cx) { + if (!cx->options().wasmBaseline() || !BaselinePlatformSupport()) { + return false; + } + bool isDisabled = false; + MOZ_ALWAYS_TRUE(BaselineDisabledByFeatures(cx, &isDisabled)); + return !isDisabled; +} + +bool wasm::IonAvailable(JSContext* cx) { + if (!cx->options().wasmIon() || !IonPlatformSupport()) { + return false; + } + bool isDisabled = false; + MOZ_ALWAYS_TRUE(IonDisabledByFeatures(cx, &isDisabled)); + return !isDisabled; +} + +bool wasm::WasmCompilerForAsmJSAvailable(JSContext* cx) { + return IonAvailable(cx); +} + +template +static inline bool Append(JSStringBuilder* reason, const char (&s)[ArrayLength], + char* sep) { + if ((*sep && !reason->append(*sep)) || !reason->append(s)) { + return false; + } + *sep = ','; + return true; +} + +bool wasm::BaselineDisabledByFeatures(JSContext* cx, bool* isDisabled, + JSStringBuilder* reason) { + // Baseline cannot be used if we are testing serialization. + bool testSerialization = WasmTestSerializationFlag(cx); + if (reason) { + char sep = 0; + if (testSerialization && !Append(reason, "testSerialization", &sep)) { + return false; + } + } + *isDisabled = testSerialization; + return true; +} + +bool wasm::IonDisabledByFeatures(JSContext* cx, bool* isDisabled, + JSStringBuilder* reason) { + // Ion has no debugging support. + bool debug = WasmDebuggerActive(cx); + if (reason) { + char sep = 0; + if (debug && !Append(reason, "debug", &sep)) { + return false; + } + } + *isDisabled = debug; + return true; +} + +bool wasm::AnyCompilerAvailable(JSContext* cx) { + return wasm::BaselineAvailable(cx) || wasm::IonAvailable(cx); +} + +// Feature predicates. These must be kept in sync with the predicates in the +// section above. +// +// The meaning of these predicates is tricky: A predicate is true for a feature +// if the feature is enabled and/or compiled-in *and* we have *at least one* +// compiler that can support the feature. Subsequent compiler selection must +// ensure that only compilers that actually support the feature are used. + +#define WASM_FEATURE(NAME, LOWER_NAME, COMPILE_PRED, COMPILER_PRED, FLAG_PRED, \ + ...) \ + bool wasm::NAME##Available(JSContext* cx) { \ + return Wasm##NAME##Flag(cx) && (COMPILER_PRED); \ + } +JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE) +#undef WASM_FEATURE + +bool wasm::IsSimdPrivilegedContext(JSContext* cx) { + // This may be slightly more lenient than we want in an ideal world, but it + // remains safe. + return cx->realm() && cx->realm()->principals() && + cx->realm()->principals()->isSystemOrAddonPrincipal(); +} + +bool wasm::SimdAvailable(JSContext* cx) { + return js::jit::JitSupportsWasmSimd(); +} + +bool wasm::ThreadsAvailable(JSContext* cx) { + return WasmThreadsFlag(cx) && AnyCompilerAvailable(cx); +} + +bool wasm::HasPlatformSupport(JSContext* cx) { +#if !MOZ_LITTLE_ENDIAN() + return false; +#else + + if (!HasJitBackend()) { + return false; + } + + if (gc::SystemPageSize() > wasm::PageSize) { + return false; + } + + if (!JitOptions.supportsUnalignedAccesses) { + return false; + } + +# ifndef __wasi__ + // WASI doesn't support signals so we don't have this function. + if (!wasm::EnsureFullSignalHandlers(cx)) { + return false; + } +# endif + + if (!jit::JitSupportsAtomics()) { + return false; + } + + // Wasm threads require 8-byte lock-free atomics. + if (!jit::AtomicOperations::isLockfree8()) { + return false; + } + + // Test only whether the compilers are supported on the hardware, not whether + // they are enabled. + return BaselinePlatformSupport() || IonPlatformSupport(); +#endif +} + +bool wasm::HasSupport(JSContext* cx) { + // If the general wasm pref is on, it's on for everything. + bool prefEnabled = cx->options().wasm(); + // If the general pref is off, check trusted principals. + if (MOZ_UNLIKELY(!prefEnabled)) { + prefEnabled = cx->options().wasmForTrustedPrinciples() && cx->realm() && + cx->realm()->principals() && + cx->realm()->principals()->isSystemOrAddonPrincipal(); + } + // Do not check for compiler availability, as that may be run-time variant. + // For HasSupport() we want a stable answer depending only on prefs. + return prefEnabled && HasPlatformSupport(cx); +} + +bool wasm::StreamingCompilationAvailable(JSContext* cx) { + // This should match EnsureStreamSupport(). + return HasSupport(cx) && AnyCompilerAvailable(cx) && + cx->runtime()->offThreadPromiseState.ref().initialized() && + CanUseExtraThreads() && cx->runtime()->consumeStreamCallback && + cx->runtime()->reportStreamErrorCallback; +} + +bool wasm::CodeCachingAvailable(JSContext* cx) { + // Fuzzilli breaks the out-of-process compilation mechanism, + // so we disable it permanently in those builds. +#ifdef FUZZING_JS_FUZZILLI + return false; +#else + + // At the moment, we require Ion support for code caching. The main reason + // for this is that wasm::CompileAndSerialize() does not have access to + // information about which optimizing compiler it should use. See comments in + // CompileAndSerialize(), below. + return StreamingCompilationAvailable(cx) && IonAvailable(cx); +#endif +} + +// ============================================================================ +// Imports + +static bool ThrowBadImportArg(JSContext* cx) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMPORT_ARG); + return false; +} + +static bool ThrowBadImportType(JSContext* cx, const CacheableName& field, + const char* str) { + UniqueChars fieldQuoted = field.toQuotedString(cx); + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMPORT_TYPE, fieldQuoted.get(), str); + return false; +} + +bool js::wasm::GetImports(JSContext* cx, const Module& module, + HandleObject importObj, ImportValues* imports) { + if (!module.imports().empty() && !importObj) { + return ThrowBadImportArg(cx); + } + + const Metadata& metadata = module.metadata(); + + uint32_t tagIndex = 0; + const TagDescVector& tags = metadata.tags; + uint32_t globalIndex = 0; + const GlobalDescVector& globals = metadata.globals; + uint32_t tableIndex = 0; + const TableDescVector& tables = metadata.tables; + for (const Import& import : module.imports()) { + RootedId moduleName(cx); + if (!import.module.toPropertyKey(cx, &moduleName)) { + return false; + } + RootedId fieldName(cx); + if (!import.field.toPropertyKey(cx, &fieldName)) { + return false; + } + + RootedValue v(cx); + if (!GetProperty(cx, importObj, importObj, moduleName, &v)) { + return false; + } + + if (!v.isObject()) { + UniqueChars moduleQuoted = import.module.toQuotedString(cx); + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMPORT_FIELD, moduleQuoted.get()); + return false; + } + + RootedObject obj(cx, &v.toObject()); + if (!GetProperty(cx, obj, obj, fieldName, &v)) { + return false; + } + + switch (import.kind) { + case DefinitionKind::Function: { + // For now reject cross-compartment wrappers. These have more + // complicated realm semantics (we use nonCCWRealm in a few places) and + // may require unwrapping to test for specific function types. + if (!IsCallable(v) || IsCrossCompartmentWrapper(&v.toObject())) { + return ThrowBadImportType(cx, import.field, "Function"); + } + + if (!imports->funcs.append(&v.toObject())) { + return false; + } + + break; + } + case DefinitionKind::Table: { + const uint32_t index = tableIndex++; + if (!v.isObject() || !v.toObject().is()) { + return ThrowBadImportType(cx, import.field, "Table"); + } + + Rooted obj(cx, &v.toObject().as()); + if (obj->table().elemType() != tables[index].elemType) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_TBL_TYPE_LINK); + return false; + } + + if (!imports->tables.append(obj)) { + return false; + } + break; + } + case DefinitionKind::Memory: { + if (!v.isObject() || !v.toObject().is()) { + return ThrowBadImportType(cx, import.field, "Memory"); + } + + MOZ_ASSERT(!imports->memory); + imports->memory = &v.toObject().as(); + break; + } + case DefinitionKind::Tag: { + const uint32_t index = tagIndex++; + if (!v.isObject() || !v.toObject().is()) { + return ThrowBadImportType(cx, import.field, "Tag"); + } + + Rooted obj(cx, &v.toObject().as()); + + // Checks whether the signature of the imported exception object matches + // the signature declared in the exception import's TagDesc. + if (obj->resultType() != tags[index].type->resultType()) { + UniqueChars fieldQuoted = import.field.toQuotedString(cx); + UniqueChars moduleQuoted = import.module.toQuotedString(cx); + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_TAG_SIG, moduleQuoted.get(), + fieldQuoted.get()); + return false; + } + + if (!imports->tagObjs.append(obj)) { + ReportOutOfMemory(cx); + return false; + } + break; + } + case DefinitionKind::Global: { + const uint32_t index = globalIndex++; + const GlobalDesc& global = globals[index]; + MOZ_ASSERT(global.importIndex() == index); + + RootedVal val(cx); + if (v.isObject() && v.toObject().is()) { + Rooted obj(cx, + &v.toObject().as()); + + if (obj->isMutable() != global.isMutable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_GLOB_MUT_LINK); + return false; + } + if (obj->type() != global.type()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_GLOB_TYPE_LINK); + return false; + } + + if (imports->globalObjs.length() <= index && + !imports->globalObjs.resize(index + 1)) { + ReportOutOfMemory(cx); + return false; + } + imports->globalObjs[index] = obj; + val = obj->val(); + } else { + if (!global.type().isRefType()) { + if (global.type() == ValType::I64 && !v.isBigInt()) { + return ThrowBadImportType(cx, import.field, "BigInt"); + } + if (global.type() != ValType::I64 && !v.isNumber()) { + return ThrowBadImportType(cx, import.field, "Number"); + } + } else { + if (!global.type().isExternRef() && !v.isObjectOrNull()) { + return ThrowBadImportType(cx, import.field, + "Object-or-null value required for " + "non-externref reference type"); + } + } + + if (global.isMutable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_GLOB_MUT_LINK); + return false; + } + + if (!Val::fromJSValue(cx, global.type(), v, &val)) { + return false; + } + } + + if (!imports->globalValues.append(val)) { + return false; + } + + break; + } + } + } + + MOZ_ASSERT(globalIndex == globals.length() || + !globals[globalIndex].isImport()); + + return true; +} + +static bool DescribeScriptedCaller(JSContext* cx, ScriptedCaller* caller, + const char* introducer) { + // Note: JS::DescribeScriptedCaller returns whether a scripted caller was + // found, not whether an error was thrown. This wrapper function converts + // back to the more ordinary false-if-error form. + + JS::AutoFilename af; + if (JS::DescribeScriptedCaller(cx, &af, &caller->line)) { + caller->filename = + FormatIntroducedFilename(af.get(), caller->line, introducer); + if (!caller->filename) { + ReportOutOfMemory(cx); + return false; + } + } + + return true; +} + +static SharedCompileArgs InitCompileArgs(JSContext* cx, + const char* introducer) { + ScriptedCaller scriptedCaller; + if (!DescribeScriptedCaller(cx, &scriptedCaller, introducer)) { + return nullptr; + } + + FeatureOptions options; + return CompileArgs::buildAndReport(cx, std::move(scriptedCaller), options); +} + +// ============================================================================ +// Testing / Fuzzing support + +bool wasm::Eval(JSContext* cx, Handle code, + HandleObject importObj, + MutableHandle instanceObj) { + if (!GlobalObject::ensureConstructor(cx, cx->global(), JSProto_WebAssembly)) { + return false; + } + + MutableBytes bytecode = cx->new_(); + if (!bytecode) { + return false; + } + + if (!bytecode->append((uint8_t*)code->dataPointerEither().unwrap(), + code->byteLength())) { + ReportOutOfMemory(cx); + return false; + } + + SharedCompileArgs compileArgs = InitCompileArgs(cx, "wasm_eval"); + if (!compileArgs) { + return false; + } + + UniqueChars error; + UniqueCharsVector warnings; + SharedModule module = + CompileBuffer(*compileArgs, *bytecode, &error, &warnings, nullptr); + if (!module) { + if (error) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_COMPILE_ERROR, error.get()); + return false; + } + ReportOutOfMemory(cx); + return false; + } + + Rooted imports(cx); + if (!GetImports(cx, *module, importObj, imports.address())) { + return false; + } + + return module->instantiate(cx, imports.get(), nullptr, instanceObj); +} + +struct MOZ_STACK_CLASS SerializeListener : JS::OptimizedEncodingListener { + // MOZ_STACK_CLASS means these can be nops. + MozExternalRefCountType MOZ_XPCOM_ABI AddRef() override { return 0; } + MozExternalRefCountType MOZ_XPCOM_ABI Release() override { return 0; } + + DebugOnly called = false; + Bytes* serialized; + explicit SerializeListener(Bytes* serialized) : serialized(serialized) {} + + void storeOptimizedEncoding(const uint8_t* bytes, size_t length) override { + MOZ_ASSERT(!called); + called = true; + if (serialized->resizeUninitialized(length)) { + memcpy(serialized->begin(), bytes, length); + } + } +}; + +bool wasm::CompileAndSerialize(JSContext* cx, const ShareableBytes& bytecode, + Bytes* serialized) { + // The caller must check that code caching is available + MOZ_ASSERT(CodeCachingAvailable(cx)); + + // Create and manually fill in compile args for code caching + MutableCompileArgs compileArgs = js_new(ScriptedCaller()); + if (!compileArgs) { + return false; + } + + // The caller has ensured CodeCachingAvailable(). Moreover, we want to ensure + // we go straight to tier-2 so that we synchronously call + // JS::OptimizedEncodingListener::storeOptimizedEncoding(). + compileArgs->baselineEnabled = false; + compileArgs->forceTiering = false; + + // We always pick Ion here, and we depend on CodeCachingAvailable() having + // determined that Ion is available, see comments at CodeCachingAvailable(). + // To do better, we need to pass information about which compiler that should + // be used into CompileAndSerialize(). + compileArgs->ionEnabled = true; + + // Select features that are enabled. This is guaranteed to be consistent with + // our compiler selection, as code caching is only available if ion is + // available, and ion is only available if it's not disabled by enabled + // features. + compileArgs->features = FeatureArgs::build(cx, FeatureOptions()); + + SerializeListener listener(serialized); + + UniqueChars error; + UniqueCharsVector warnings; + SharedModule module = + CompileBuffer(*compileArgs, bytecode, &error, &warnings, &listener); + if (!module) { + fprintf(stderr, "Compilation error: %s\n", error ? error.get() : "oom"); + return false; + } + + MOZ_ASSERT(module->code().hasTier(Tier::Serialized)); + MOZ_ASSERT(listener.called); + return !listener.serialized->empty(); +} + +bool wasm::DeserializeModule(JSContext* cx, const Bytes& serialized, + MutableHandleObject moduleObj) { + MutableModule module = + Module::deserialize(serialized.begin(), serialized.length()); + if (!module) { + ReportOutOfMemory(cx); + return false; + } + + moduleObj.set(module->createObject(cx)); + return !!moduleObj; +} + +// ============================================================================ +// Common functions + +// '[EnforceRange] unsigned long' types are coerced with +// ConvertToInt(v, 32, 'unsigned') +// defined in Web IDL Section 3.2.4.9. +// +// This just generalizes that to an arbitrary limit that is representable as an +// integer in double form. + +static bool EnforceRange(JSContext* cx, HandleValue v, const char* kind, + const char* noun, uint64_t max, uint64_t* val) { + // Step 4. + double x; + if (!ToNumber(cx, v, &x)) { + return false; + } + + // Step 5. + if (mozilla::IsNegativeZero(x)) { + x = 0.0; + } + + // Step 6.1. + if (!std::isfinite(x)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_ENFORCE_RANGE, kind, noun); + return false; + } + + // Step 6.2. + x = JS::ToInteger(x); + + // Step 6.3. + if (x < 0 || x > double(max)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_ENFORCE_RANGE, kind, noun); + return false; + } + + *val = uint64_t(x); + MOZ_ASSERT(double(*val) == x); + return true; +} + +static bool EnforceRangeU32(JSContext* cx, HandleValue v, const char* kind, + const char* noun, uint32_t* u32) { + uint64_t u64 = 0; + if (!EnforceRange(cx, v, kind, noun, uint64_t(UINT32_MAX), &u64)) { + return false; + } + *u32 = uint32_t(u64); + return true; +} + +static bool EnforceRangeU64(JSContext* cx, HandleValue v, const char* kind, + const char* noun, uint64_t* u64) { + // The max is Number.MAX_SAFE_INTEGER + return EnforceRange(cx, v, kind, noun, (1LL << 53) - 1, u64); +} + +static bool GetLimit(JSContext* cx, HandleObject obj, const char* name, + const char* noun, const char* msg, uint32_t range, + bool* found, uint64_t* value) { + JSAtom* atom = Atomize(cx, name, strlen(name)); + if (!atom) { + return false; + } + RootedId id(cx, AtomToId(atom)); + + RootedValue val(cx); + if (!GetProperty(cx, obj, obj, id, &val)) { + return false; + } + + if (val.isUndefined()) { + *found = false; + return true; + } + *found = true; + // The range can be greater than 53, but then the logic in EnforceRange has to + // change to avoid precision loss. + MOZ_ASSERT(range < 54); + return EnforceRange(cx, val, noun, msg, (uint64_t(1) << range) - 1, value); +} + +static bool GetLimits(JSContext* cx, HandleObject obj, LimitsKind kind, + Limits* limits) { + limits->indexType = IndexType::I32; + + // Memory limits may specify an alternate index type, and we need this to + // check the ranges for initial and maximum, so look for the index type first. + if (kind == LimitsKind::Memory) { +#ifdef ENABLE_WASM_MEMORY64 + // Get the index type field + JSAtom* indexTypeAtom = Atomize(cx, "index", strlen("index")); + if (!indexTypeAtom) { + return false; + } + RootedId indexTypeId(cx, AtomToId(indexTypeAtom)); + + RootedValue indexTypeVal(cx); + if (!GetProperty(cx, obj, obj, indexTypeId, &indexTypeVal)) { + return false; + } + + // The index type has a default value + if (!indexTypeVal.isUndefined()) { + if (!ToIndexType(cx, indexTypeVal, &limits->indexType)) { + return false; + } + + if (limits->indexType == IndexType::I64 && !Memory64Available(cx)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_NO_MEM64_LINK); + return false; + } + } +#endif + } + + const char* noun = (kind == LimitsKind::Memory ? "Memory" : "Table"); + // 2^48 is a valid value, so the range goes to 49 bits. Values above 2^48 are + // filtered later, just as values above 2^16 are filtered for mem32. + const uint32_t range = limits->indexType == IndexType::I32 ? 32 : 49; + uint64_t limit = 0; + + bool haveInitial = false; + if (!GetLimit(cx, obj, "initial", noun, "initial size", range, &haveInitial, + &limit)) { + return false; + } + if (haveInitial) { + limits->initial = limit; + } + + bool haveMinimum = false; +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + if (!GetLimit(cx, obj, "minimum", noun, "initial size", range, &haveMinimum, + &limit)) { + return false; + } + if (haveMinimum) { + limits->initial = limit; + } +#endif + + if (!(haveInitial || haveMinimum)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_MISSING_REQUIRED, "initial"); + return false; + } + if (haveInitial && haveMinimum) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_SUPPLY_ONLY_ONE, "minimum", "initial"); + return false; + } + + bool haveMaximum = false; + if (!GetLimit(cx, obj, "maximum", noun, "maximum size", range, &haveMaximum, + &limit)) { + return false; + } + if (haveMaximum) { + limits->maximum = Some(limit); + } + + limits->shared = Shareable::False; + + // Memory limits may be shared. + if (kind == LimitsKind::Memory) { + // Get the shared field + JSAtom* sharedAtom = Atomize(cx, "shared", strlen("shared")); + if (!sharedAtom) { + return false; + } + RootedId sharedId(cx, AtomToId(sharedAtom)); + + RootedValue sharedVal(cx); + if (!GetProperty(cx, obj, obj, sharedId, &sharedVal)) { + return false; + } + + // shared's default value is false, which is already the value set above. + if (!sharedVal.isUndefined()) { + limits->shared = + ToBoolean(sharedVal) ? Shareable::True : Shareable::False; + + if (limits->shared == Shareable::True) { + if (!haveMaximum) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_MISSING_MAXIMUM, noun); + return false; + } + + if (!cx->realm() + ->creationOptions() + .getSharedMemoryAndAtomicsEnabled()) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_NO_SHMEM_LINK); + return false; + } + } + } + } + + return true; +} + +static bool CheckLimits(JSContext* cx, uint64_t maximumField, LimitsKind kind, + Limits* limits) { + const char* noun = (kind == LimitsKind::Memory ? "Memory" : "Table"); + + if (limits->initial > maximumField) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_RANGE, + noun, "initial size"); + return false; + } + + if (limits->maximum.isSome() && + (*limits->maximum > maximumField || limits->initial > *limits->maximum)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_RANGE, + noun, "maximum size"); + return false; + } + return true; +} + +template +static JSObject* CreateWasmConstructor(JSContext* cx, JSProtoKey key) { + Rooted className(cx, Atomize(cx, name, strlen(name))); + if (!className) { + return nullptr; + } + + return NewNativeConstructor(cx, Class::construct, 1, className); +} + +static JSObject* GetWasmConstructorPrototype(JSContext* cx, + const CallArgs& callArgs, + JSProtoKey key) { + RootedObject proto(cx); + if (!GetPrototypeFromBuiltinConstructor(cx, callArgs, key, &proto)) { + return nullptr; + } + if (!proto) { + proto = GlobalObject::getOrCreatePrototype(cx, key); + } + return proto; +} + +[[nodiscard]] static bool ParseValTypes(JSContext* cx, HandleValue src, + ValTypeVector& dest) { + JS::ForOfIterator iterator(cx); + + if (!iterator.init(src, JS::ForOfIterator::ThrowOnNonIterable)) { + return false; + } + + RootedValue nextParam(cx); + while (true) { + bool done; + if (!iterator.next(&nextParam, &done)) { + return false; + } + if (done) { + break; + } + + ValType valType; + if (!ToValType(cx, nextParam, &valType) || !dest.append(valType)) { + return false; + } + } + return true; +} + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS +static JSString* UTF8CharsToString(JSContext* cx, const char* chars) { + return NewStringCopyUTF8Z(cx, JS::ConstUTF8CharsZ(chars, strlen(chars))); +} + +[[nodiscard]] static JSObject* ValTypesToArray(JSContext* cx, + const ValTypeVector& valTypes) { + Rooted arrayObj(cx, NewDenseEmptyArray(cx)); + if (!arrayObj) { + return nullptr; + } + for (ValType valType : valTypes) { + RootedString type(cx, + UTF8CharsToString(cx, ToString(valType, nullptr).get())); + if (!type) { + return nullptr; + } + if (!NewbornArrayPush(cx, arrayObj, StringValue(type))) { + return nullptr; + } + } + return arrayObj; +} + +static JSObject* FuncTypeToObject(JSContext* cx, const FuncType& type) { + Rooted props(cx, IdValueVector(cx)); + + RootedObject parametersObj(cx, ValTypesToArray(cx, type.args())); + if (!parametersObj || + !props.append(IdValuePair(NameToId(cx->names().parameters), + ObjectValue(*parametersObj)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + RootedObject resultsObj(cx, ValTypesToArray(cx, type.results())); + if (!resultsObj || !props.append(IdValuePair(NameToId(cx->names().results), + ObjectValue(*resultsObj)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + return NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); +} + +static JSObject* TableTypeToObject(JSContext* cx, RefType type, + uint32_t initial, Maybe maximum) { + Rooted props(cx, IdValueVector(cx)); + + RootedString elementType( + cx, UTF8CharsToString(cx, ToString(type, nullptr).get())); + if (!elementType || !props.append(IdValuePair(NameToId(cx->names().element), + StringValue(elementType)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + if (maximum.isSome()) { + if (!props.append(IdValuePair(NameToId(cx->names().maximum), + NumberValue(maximum.value())))) { + ReportOutOfMemory(cx); + return nullptr; + } + } + + if (!props.append( + IdValuePair(NameToId(cx->names().minimum), NumberValue(initial)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + return NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); +} + +static JSObject* MemoryTypeToObject(JSContext* cx, bool shared, + wasm::IndexType indexType, + wasm::Pages minPages, + Maybe maxPages) { + Rooted props(cx, IdValueVector(cx)); + if (maxPages) { + double maxPagesNum; + if (indexType == IndexType::I32) { + maxPagesNum = double(mozilla::AssertedCast(maxPages->value())); + } else { + // The maximum number of pages is 2^48. + maxPagesNum = double(maxPages->value()); + } + if (!props.append(IdValuePair(NameToId(cx->names().maximum), + NumberValue(maxPagesNum)))) { + ReportOutOfMemory(cx); + return nullptr; + } + } + + double minPagesNum; + if (indexType == IndexType::I32) { + minPagesNum = double(mozilla::AssertedCast(minPages.value())); + } else { + minPagesNum = double(minPages.value()); + } + if (!props.append(IdValuePair(NameToId(cx->names().minimum), + NumberValue(minPagesNum)))) { + ReportOutOfMemory(cx); + return nullptr; + } + +# ifdef ENABLE_WASM_MEMORY64 + RootedString it( + cx, JS_NewStringCopyZ(cx, indexType == IndexType::I32 ? "i32" : "i64")); + if (!it) { + return nullptr; + } + if (!props.append( + IdValuePair(NameToId(cx->names().index), StringValue(it)))) { + ReportOutOfMemory(cx); + return nullptr; + } +# endif + + if (!props.append( + IdValuePair(NameToId(cx->names().shared), BooleanValue(shared)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + return NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); +} + +static JSObject* GlobalTypeToObject(JSContext* cx, ValType type, + bool isMutable) { + Rooted props(cx, IdValueVector(cx)); + + if (!props.append(IdValuePair(NameToId(cx->names().mutable_), + BooleanValue(isMutable)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + RootedString valueType(cx, + UTF8CharsToString(cx, ToString(type, nullptr).get())); + if (!valueType || !props.append(IdValuePair(NameToId(cx->names().value), + StringValue(valueType)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + return NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); +} + +static JSObject* TagTypeToObject(JSContext* cx, + const wasm::ValTypeVector& params) { + Rooted props(cx, IdValueVector(cx)); + + RootedObject parametersObj(cx, ValTypesToArray(cx, params)); + if (!parametersObj || + !props.append(IdValuePair(NameToId(cx->names().parameters), + ObjectValue(*parametersObj)))) { + ReportOutOfMemory(cx); + return nullptr; + } + + return NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); +} +#endif // ENABLE_WASM_TYPE_REFLECTIONS + +// ============================================================================ +// WebAssembly.Module class and methods + +const JSClassOps WasmModuleObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmModuleObject::finalize, // finalize + nullptr, // call + nullptr, // construct + nullptr, // trace +}; + +const JSClass WasmModuleObject::class_ = { + "WebAssembly.Module", + JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_HAS_RESERVED_SLOTS(WasmModuleObject::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &WasmModuleObject::classOps_, + &WasmModuleObject::classSpec_, +}; + +const JSClass& WasmModuleObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmModuleName[] = "Module"; + +const ClassSpec WasmModuleObject::classSpec_ = { + CreateWasmConstructor, + GenericCreatePrototype, + WasmModuleObject::static_methods, + nullptr, + WasmModuleObject::methods, + WasmModuleObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +const JSPropertySpec WasmModuleObject::properties[] = { + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Module", JSPROP_READONLY), + JS_PS_END}; + +const JSFunctionSpec WasmModuleObject::methods[] = {JS_FS_END}; + +const JSFunctionSpec WasmModuleObject::static_methods[] = { + JS_FN("imports", WasmModuleObject::imports, 1, JSPROP_ENUMERATE), + JS_FN("exports", WasmModuleObject::exports, 1, JSPROP_ENUMERATE), + JS_FN("customSections", WasmModuleObject::customSections, 2, + JSPROP_ENUMERATE), + JS_FS_END}; + +/* static */ +void WasmModuleObject::finalize(JS::GCContext* gcx, JSObject* obj) { + const Module& module = obj->as().module(); + obj->zone()->decJitMemory(module.codeLength(module.code().stableTier())); + gcx->release(obj, &module, module.gcMallocBytesExcludingCode(), + MemoryUse::WasmModule); +} + +static bool IsModuleObject(JSObject* obj, const Module** module) { + WasmModuleObject* mobj = obj->maybeUnwrapIf(); + if (!mobj) { + return false; + } + + *module = &mobj->module(); + return true; +} + +static bool GetModuleArg(JSContext* cx, CallArgs args, uint32_t numRequired, + const char* name, const Module** module) { + if (!args.requireAtLeast(cx, name, numRequired)) { + return false; + } + + if (!args[0].isObject() || !IsModuleObject(&args[0].toObject(), module)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_MOD_ARG); + return false; + } + + return true; +} + +struct KindNames { + Rooted kind; + Rooted table; + Rooted memory; + Rooted tag; + Rooted type; + + explicit KindNames(JSContext* cx) + : kind(cx), table(cx), memory(cx), tag(cx), type(cx) {} +}; + +static bool InitKindNames(JSContext* cx, KindNames* names) { + JSAtom* kind = Atomize(cx, "kind", strlen("kind")); + if (!kind) { + return false; + } + names->kind = kind->asPropertyName(); + + JSAtom* table = Atomize(cx, "table", strlen("table")); + if (!table) { + return false; + } + names->table = table->asPropertyName(); + + JSAtom* memory = Atomize(cx, "memory", strlen("memory")); + if (!memory) { + return false; + } + names->memory = memory->asPropertyName(); + + JSAtom* tag = Atomize(cx, "tag", strlen("tag")); + if (!tag) { + return false; + } + names->tag = tag->asPropertyName(); + + JSAtom* type = Atomize(cx, "type", strlen("type")); + if (!type) { + return false; + } + names->type = type->asPropertyName(); + + return true; +} + +static JSString* KindToString(JSContext* cx, const KindNames& names, + DefinitionKind kind) { + switch (kind) { + case DefinitionKind::Function: + return cx->names().function; + case DefinitionKind::Table: + return names.table; + case DefinitionKind::Memory: + return names.memory; + case DefinitionKind::Global: + return cx->names().global; + case DefinitionKind::Tag: + return names.tag; + } + + MOZ_CRASH("invalid kind"); +} + +/* static */ +bool WasmModuleObject::imports(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + const Module* module; + if (!GetModuleArg(cx, args, 1, "WebAssembly.Module.imports", &module)) { + return false; + } + + KindNames names(cx); + if (!InitKindNames(cx, &names)) { + return false; + } + + RootedValueVector elems(cx); + if (!elems.reserve(module->imports().length())) { + return false; + } + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + const Metadata& metadata = module->metadata(); + const MetadataTier& metadataTier = + module->metadata(module->code().stableTier()); + + size_t numFuncImport = 0; + size_t numMemoryImport = 0; + size_t numGlobalImport = 0; + size_t numTableImport = 0; + size_t numTagImport = 0; +#endif // ENABLE_WASM_TYPE_REFLECTIONS + + for (const Import& import : module->imports()) { + Rooted props(cx, IdValueVector(cx)); + if (!props.reserve(3)) { + return false; + } + + JSString* moduleStr = import.module.toAtom(cx); + if (!moduleStr) { + return false; + } + props.infallibleAppend( + IdValuePair(NameToId(cx->names().module), StringValue(moduleStr))); + + JSString* nameStr = import.field.toAtom(cx); + if (!nameStr) { + return false; + } + props.infallibleAppend( + IdValuePair(NameToId(cx->names().name), StringValue(nameStr))); + + JSString* kindStr = KindToString(cx, names, import.kind); + if (!kindStr) { + return false; + } + props.infallibleAppend( + IdValuePair(NameToId(names.kind), StringValue(kindStr))); + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + RootedObject typeObj(cx); + switch (import.kind) { + case DefinitionKind::Function: { + size_t funcIndex = numFuncImport++; + const FuncType& funcType = + metadata.getFuncImportType(metadataTier.funcImports[funcIndex]); + typeObj = FuncTypeToObject(cx, funcType); + break; + } + case DefinitionKind::Table: { + size_t tableIndex = numTableImport++; + const TableDesc& table = metadata.tables[tableIndex]; + typeObj = TableTypeToObject(cx, table.elemType, table.initialLength, + table.maximumLength); + break; + } + case DefinitionKind::Memory: { + DebugOnly memoryIndex = numMemoryImport++; + MOZ_ASSERT(memoryIndex == 0); + const MemoryDesc& memory = *metadata.memory; + typeObj = + MemoryTypeToObject(cx, memory.isShared(), memory.indexType(), + memory.initialPages(), memory.maximumPages()); + break; + } + case DefinitionKind::Global: { + size_t globalIndex = numGlobalImport++; + const GlobalDesc& global = metadata.globals[globalIndex]; + typeObj = GlobalTypeToObject(cx, global.type(), global.isMutable()); + break; + } + case DefinitionKind::Tag: { + size_t tagIndex = numTagImport++; + const TagDesc& tag = metadata.tags[tagIndex]; + typeObj = TagTypeToObject(cx, tag.type->argTypes_); + break; + } + } + + if (!typeObj || !props.append(IdValuePair(NameToId(names.type), + ObjectValue(*typeObj)))) { + ReportOutOfMemory(cx); + return false; + } +#endif // ENABLE_WASM_TYPE_REFLECTIONS + + JSObject* obj = + NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); + if (!obj) { + return false; + } + + elems.infallibleAppend(ObjectValue(*obj)); + } + + JSObject* arr = NewDenseCopiedArray(cx, elems.length(), elems.begin()); + if (!arr) { + return false; + } + + args.rval().setObject(*arr); + return true; +} + +/* static */ +bool WasmModuleObject::exports(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + const Module* module; + if (!GetModuleArg(cx, args, 1, "WebAssembly.Module.exports", &module)) { + return false; + } + + KindNames names(cx); + if (!InitKindNames(cx, &names)) { + return false; + } + + RootedValueVector elems(cx); + if (!elems.reserve(module->exports().length())) { + return false; + } + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + const Metadata& metadata = module->metadata(); + const MetadataTier& metadataTier = + module->metadata(module->code().stableTier()); +#endif // ENABLE_WASM_TYPE_REFLECTIONS + + for (const Export& exp : module->exports()) { + Rooted props(cx, IdValueVector(cx)); + if (!props.reserve(2)) { + return false; + } + + JSString* nameStr = exp.fieldName().toAtom(cx); + if (!nameStr) { + return false; + } + props.infallibleAppend( + IdValuePair(NameToId(cx->names().name), StringValue(nameStr))); + + JSString* kindStr = KindToString(cx, names, exp.kind()); + if (!kindStr) { + return false; + } + props.infallibleAppend( + IdValuePair(NameToId(names.kind), StringValue(kindStr))); + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + RootedObject typeObj(cx); + switch (exp.kind()) { + case DefinitionKind::Function: { + const FuncExport& fe = metadataTier.lookupFuncExport(exp.funcIndex()); + const FuncType& funcType = metadata.getFuncExportType(fe); + typeObj = FuncTypeToObject(cx, funcType); + break; + } + case DefinitionKind::Table: { + const TableDesc& table = metadata.tables[exp.tableIndex()]; + typeObj = TableTypeToObject(cx, table.elemType, table.initialLength, + table.maximumLength); + break; + } + case DefinitionKind::Memory: { + const MemoryDesc& memory = *metadata.memory; + typeObj = + MemoryTypeToObject(cx, memory.isShared(), memory.indexType(), + memory.initialPages(), memory.maximumPages()); + break; + } + case DefinitionKind::Global: { + const GlobalDesc& global = metadata.globals[exp.globalIndex()]; + typeObj = GlobalTypeToObject(cx, global.type(), global.isMutable()); + break; + } + case DefinitionKind::Tag: { + const TagDesc& tag = metadata.tags[exp.tagIndex()]; + typeObj = TagTypeToObject(cx, tag.type->argTypes_); + break; + } + } + + if (!typeObj || !props.append(IdValuePair(NameToId(names.type), + ObjectValue(*typeObj)))) { + ReportOutOfMemory(cx); + return false; + } +#endif // ENABLE_WASM_TYPE_REFLECTIONS + + JSObject* obj = + NewPlainObjectWithUniqueNames(cx, props.begin(), props.length()); + if (!obj) { + return false; + } + + elems.infallibleAppend(ObjectValue(*obj)); + } + + JSObject* arr = NewDenseCopiedArray(cx, elems.length(), elems.begin()); + if (!arr) { + return false; + } + + args.rval().setObject(*arr); + return true; +} + +/* static */ +bool WasmModuleObject::customSections(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + const Module* module; + if (!GetModuleArg(cx, args, 2, "WebAssembly.Module.customSections", + &module)) { + return false; + } + + Vector name(cx); + { + RootedString str(cx, ToString(cx, args.get(1))); + if (!str) { + return false; + } + + Rooted linear(cx, str->ensureLinear(cx)); + if (!linear) { + return false; + } + + if (!name.initLengthUninitialized( + JS::GetDeflatedUTF8StringLength(linear))) { + return false; + } + + (void)JS::DeflateStringToUTF8Buffer(linear, + Span(name.begin(), name.length())); + } + + RootedValueVector elems(cx); + RootedArrayBufferObject buf(cx); + for (const CustomSection& cs : module->customSections()) { + if (name.length() != cs.name.length()) { + continue; + } + if (memcmp(name.begin(), cs.name.begin(), name.length()) != 0) { + continue; + } + + buf = ArrayBufferObject::createZeroed(cx, cs.payload->length()); + if (!buf) { + return false; + } + + memcpy(buf->dataPointer(), cs.payload->begin(), cs.payload->length()); + if (!elems.append(ObjectValue(*buf))) { + return false; + } + } + + JSObject* arr = NewDenseCopiedArray(cx, elems.length(), elems.begin()); + if (!arr) { + return false; + } + + args.rval().setObject(*arr); + return true; +} + +/* static */ +WasmModuleObject* WasmModuleObject::create(JSContext* cx, const Module& module, + HandleObject proto) { + AutoSetNewObjectMetadata metadata(cx); + auto* obj = NewObjectWithGivenProto(cx, proto); + if (!obj) { + return nullptr; + } + + // The pipeline state on some architectures may retain stale instructions + // even after we invalidate the instruction cache. There is no generally + // available method to broadcast this pipeline flush to all threads after + // we've compiled new code, so conservatively perform one here when we're + // receiving a module that may have been compiled from another thread. + // + // The cost of this flush is expected to minimal enough to not be worth + // optimizing away in the case the module was compiled on this thread. + jit::FlushExecutionContext(); + + // This accounts for module allocation size (excluding code which is handled + // separately - see below). This assumes that the size of associated data + // doesn't change for the life of the WasmModuleObject. The size is counted + // once per WasmModuleObject referencing a Module. + InitReservedSlot(obj, MODULE_SLOT, const_cast(&module), + module.gcMallocBytesExcludingCode(), MemoryUse::WasmModule); + module.AddRef(); + + // Bug 1569888: We account for the first tier here; the second tier, if + // different, also needs to be accounted for. + cx->zone()->incJitMemory(module.codeLength(module.code().stableTier())); + return obj; +} + +static bool GetBufferSource(JSContext* cx, JSObject* obj, unsigned errorNumber, + MutableBytes* bytecode) { + *bytecode = cx->new_(); + if (!*bytecode) { + return false; + } + + JSObject* unwrapped = CheckedUnwrapStatic(obj); + + SharedMem dataPointer; + size_t byteLength; + if (!unwrapped || !IsBufferSource(unwrapped, &dataPointer, &byteLength)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, errorNumber); + return false; + } + + if (!(*bytecode)->append(dataPointer.unwrap(), byteLength)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +static bool ReportCompileWarnings(JSContext* cx, + const UniqueCharsVector& warnings) { + // Avoid spamming the console. + size_t numWarnings = std::min(warnings.length(), 3); + + for (size_t i = 0; i < numWarnings; i++) { + if (!WarnNumberASCII(cx, JSMSG_WASM_COMPILE_WARNING, warnings[i].get())) { + return false; + } + } + + if (warnings.length() > numWarnings) { + if (!WarnNumberASCII(cx, JSMSG_WASM_COMPILE_WARNING, + "other warnings suppressed")) { + return false; + } + } + + return true; +} + +/* static */ +bool WasmModuleObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs callArgs = CallArgsFromVp(argc, vp); + + Log(cx, "sync new Module() started"); + + if (!ThrowIfNotConstructing(cx, callArgs, "Module")) { + return false; + } + + if (!cx->isRuntimeCodeGenEnabled(JS::RuntimeCode::WASM, nullptr)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_CSP_BLOCKED_WASM, "WebAssembly.Module"); + return false; + } + + if (!callArgs.requireAtLeast(cx, "WebAssembly.Module", 1)) { + return false; + } + + if (!callArgs[0].isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_BUF_ARG); + return false; + } + + MutableBytes bytecode; + if (!GetBufferSource(cx, &callArgs[0].toObject(), JSMSG_WASM_BAD_BUF_ARG, + &bytecode)) { + return false; + } + + SharedCompileArgs compileArgs = InitCompileArgs(cx, "WebAssembly.Module"); + if (!compileArgs) { + return false; + } + + UniqueChars error; + UniqueCharsVector warnings; + SharedModule module = + CompileBuffer(*compileArgs, *bytecode, &error, &warnings, nullptr); + + if (!ReportCompileWarnings(cx, warnings)) { + return false; + } + if (!module) { + if (error) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_COMPILE_ERROR, error.get()); + return false; + } + ReportOutOfMemory(cx); + return false; + } + + RootedObject proto( + cx, GetWasmConstructorPrototype(cx, callArgs, JSProto_WasmModule)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + RootedObject moduleObj(cx, WasmModuleObject::create(cx, *module, proto)); + if (!moduleObj) { + return false; + } + + Log(cx, "sync new Module() succeded"); + + callArgs.rval().setObject(*moduleObj); + return true; +} + +const Module& WasmModuleObject::module() const { + MOZ_ASSERT(is()); + return *(const Module*)getReservedSlot(MODULE_SLOT).toPrivate(); +} + +// ============================================================================ +// WebAssembly.Instance class and methods + +const JSClassOps WasmInstanceObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmInstanceObject::finalize, // finalize + nullptr, // call + nullptr, // construct + WasmInstanceObject::trace, // trace +}; + +const JSClass WasmInstanceObject::class_ = { + "WebAssembly.Instance", + JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_HAS_RESERVED_SLOTS(WasmInstanceObject::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &WasmInstanceObject::classOps_, + &WasmInstanceObject::classSpec_, +}; + +const JSClass& WasmInstanceObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmInstanceName[] = "Instance"; + +const ClassSpec WasmInstanceObject::classSpec_ = { + CreateWasmConstructor, + GenericCreatePrototype, + WasmInstanceObject::static_methods, + nullptr, + WasmInstanceObject::methods, + WasmInstanceObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +static bool IsInstance(HandleValue v) { + return v.isObject() && v.toObject().is(); +} + +/* static */ +bool WasmInstanceObject::exportsGetterImpl(JSContext* cx, + const CallArgs& args) { + args.rval().setObject( + args.thisv().toObject().as().exportsObj()); + return true; +} + +/* static */ +bool WasmInstanceObject::exportsGetter(JSContext* cx, unsigned argc, + Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +const JSPropertySpec WasmInstanceObject::properties[] = { + JS_PSG("exports", WasmInstanceObject::exportsGetter, JSPROP_ENUMERATE), + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Instance", JSPROP_READONLY), + JS_PS_END}; + +const JSFunctionSpec WasmInstanceObject::methods[] = {JS_FS_END}; + +const JSFunctionSpec WasmInstanceObject::static_methods[] = {JS_FS_END}; + +bool WasmInstanceObject::isNewborn() const { + MOZ_ASSERT(is()); + return getReservedSlot(INSTANCE_SLOT).isUndefined(); +} + +// WeakScopeMap maps from function index to js::Scope. This maps is weak +// to avoid holding scope objects alive. The scopes are normally created +// during debugging. +// +// This is defined here in order to avoid recursive dependency between +// WasmJS.h and Scope.h. +using WasmFunctionScopeMap = + JS::WeakCache, + DefaultHasher, CellAllocPolicy>>; +class WasmInstanceObject::UnspecifiedScopeMap { + public: + WasmFunctionScopeMap& asWasmFunctionScopeMap() { + return *(WasmFunctionScopeMap*)this; + } +}; + +/* static */ +void WasmInstanceObject::finalize(JS::GCContext* gcx, JSObject* obj) { + WasmInstanceObject& instance = obj->as(); + gcx->delete_(obj, &instance.exports(), MemoryUse::WasmInstanceExports); + gcx->delete_(obj, &instance.scopes().asWasmFunctionScopeMap(), + MemoryUse::WasmInstanceScopes); + gcx->delete_(obj, &instance.indirectGlobals(), + MemoryUse::WasmInstanceGlobals); + if (!instance.isNewborn()) { + if (instance.instance().debugEnabled()) { + instance.instance().debug().finalize(gcx); + } + Instance::destroy(&instance.instance()); + gcx->removeCellMemory(obj, sizeof(Instance), + MemoryUse::WasmInstanceInstance); + } +} + +/* static */ +void WasmInstanceObject::trace(JSTracer* trc, JSObject* obj) { + WasmInstanceObject& instanceObj = obj->as(); + instanceObj.exports().trace(trc); + instanceObj.indirectGlobals().trace(trc); + if (!instanceObj.isNewborn()) { + instanceObj.instance().tracePrivate(trc); + } +} + +/* static */ +WasmInstanceObject* WasmInstanceObject::create( + JSContext* cx, const SharedCode& code, + const DataSegmentVector& dataSegments, + const ElemSegmentVector& elemSegments, uint32_t instanceDataLength, + Handle memory, SharedTableVector&& tables, + const JSObjectVector& funcImports, const GlobalDescVector& globals, + const ValVector& globalImportValues, + const WasmGlobalObjectVector& globalObjs, + const WasmTagObjectVector& tagObjs, HandleObject proto, + UniqueDebugState maybeDebug) { + Rooted> exports(cx, + js::MakeUnique(cx->zone())); + if (!exports) { + ReportOutOfMemory(cx); + return nullptr; + } + + UniquePtr scopes = + js::MakeUnique(cx->zone(), cx->zone()); + if (!scopes) { + ReportOutOfMemory(cx); + return nullptr; + } + // Note that `scopes` is a WeakCache, auto-linked into a sweep list on the + // Zone, and so does not require rooting. + + uint32_t indirectGlobals = 0; + + for (uint32_t i = 0; i < globalObjs.length(); i++) { + if (globalObjs[i] && globals[i].isIndirect()) { + indirectGlobals++; + } + } + + Rooted> indirectGlobalObjs( + cx, js::MakeUnique(cx->zone())); + if (!indirectGlobalObjs || !indirectGlobalObjs->resize(indirectGlobals)) { + ReportOutOfMemory(cx); + return nullptr; + } + + { + uint32_t next = 0; + for (uint32_t i = 0; i < globalObjs.length(); i++) { + if (globalObjs[i] && globals[i].isIndirect()) { + (*indirectGlobalObjs)[next++] = globalObjs[i]; + } + } + } + + Instance* instance = nullptr; + Rooted obj(cx); + + { + // We must delay creating metadata for this object until after all its + // slots have been initialized. We must also create the metadata before + // calling Instance::init as that may allocate new objects. + AutoSetNewObjectMetadata metadata(cx); + obj = NewObjectWithGivenProto(cx, proto); + if (!obj) { + return nullptr; + } + + MOZ_ASSERT(obj->isTenured(), "assumed by WasmTableObject write barriers"); + + // Finalization assumes these slots are always initialized: + InitReservedSlot(obj, EXPORTS_SLOT, exports.release(), + MemoryUse::WasmInstanceExports); + + InitReservedSlot(obj, SCOPES_SLOT, scopes.release(), + MemoryUse::WasmInstanceScopes); + + InitReservedSlot(obj, GLOBALS_SLOT, indirectGlobalObjs.release(), + MemoryUse::WasmInstanceGlobals); + + obj->initReservedSlot(INSTANCE_SCOPE_SLOT, UndefinedValue()); + + // The INSTANCE_SLOT may not be initialized if Instance allocation fails, + // leading to an observable "newborn" state in tracing/finalization. + MOZ_ASSERT(obj->isNewborn()); + + // Create this just before constructing Instance to avoid rooting hazards. + instance = Instance::create(cx, obj, code, instanceDataLength, memory, + std::move(tables), std::move(maybeDebug)); + if (!instance) { + return nullptr; + } + + InitReservedSlot(obj, INSTANCE_SLOT, instance, + MemoryUse::WasmInstanceInstance); + MOZ_ASSERT(!obj->isNewborn()); + } + + if (!instance->init(cx, funcImports, globalImportValues, globalObjs, tagObjs, + dataSegments, elemSegments)) { + return nullptr; + } + + return obj; +} + +void WasmInstanceObject::initExportsObj(JSObject& exportsObj) { + MOZ_ASSERT(getReservedSlot(EXPORTS_OBJ_SLOT).isUndefined()); + setReservedSlot(EXPORTS_OBJ_SLOT, ObjectValue(exportsObj)); +} + +static bool GetImportArg(JSContext* cx, CallArgs callArgs, + MutableHandleObject importObj) { + if (!callArgs.get(1).isUndefined()) { + if (!callArgs[1].isObject()) { + return ThrowBadImportArg(cx); + } + importObj.set(&callArgs[1].toObject()); + } + return true; +} + +/* static */ +bool WasmInstanceObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + Log(cx, "sync new Instance() started"); + + if (!ThrowIfNotConstructing(cx, args, "Instance")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Instance", 1)) { + return false; + } + + const Module* module; + if (!args[0].isObject() || !IsModuleObject(&args[0].toObject(), &module)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_MOD_ARG); + return false; + } + + RootedObject importObj(cx); + if (!GetImportArg(cx, args, &importObj)) { + return false; + } + + RootedObject proto( + cx, GetWasmConstructorPrototype(cx, args, JSProto_WasmInstance)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + Rooted imports(cx); + if (!GetImports(cx, *module, importObj, imports.address())) { + return false; + } + + Rooted instanceObj(cx); + if (!module->instantiate(cx, imports.get(), proto, &instanceObj)) { + return false; + } + + Log(cx, "sync new Instance() succeeded"); + + args.rval().setObject(*instanceObj); + return true; +} + +Instance& WasmInstanceObject::instance() const { + MOZ_ASSERT(!isNewborn()); + return *(Instance*)getReservedSlot(INSTANCE_SLOT).toPrivate(); +} + +JSObject& WasmInstanceObject::exportsObj() const { + return getReservedSlot(EXPORTS_OBJ_SLOT).toObject(); +} + +WasmInstanceObject::ExportMap& WasmInstanceObject::exports() const { + return *(ExportMap*)getReservedSlot(EXPORTS_SLOT).toPrivate(); +} + +WasmInstanceObject::UnspecifiedScopeMap& WasmInstanceObject::scopes() const { + return *(UnspecifiedScopeMap*)(getReservedSlot(SCOPES_SLOT).toPrivate()); +} + +WasmInstanceObject::GlobalObjectVector& WasmInstanceObject::indirectGlobals() + const { + return *(GlobalObjectVector*)getReservedSlot(GLOBALS_SLOT).toPrivate(); +} + +static bool WasmCall(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + RootedFunction callee(cx, &args.callee().as()); + + Instance& instance = ExportedFunctionToInstance(callee); + uint32_t funcIndex = ExportedFunctionToFuncIndex(callee); + return instance.callExport(cx, funcIndex, args); +} + +/* + * [SMDOC] Exported wasm functions and the jit-entry stubs + * + * ## The kinds of exported functions + * + * There are several kinds of exported wasm functions. /Explicitly/ exported + * functions are: + * + * - any wasm function exported via the export section + * - any asm.js export + * - the module start function + * + * There are also /implicitly/ exported functions, these are the functions whose + * indices in the module are referenced outside the code segment, eg, in element + * segments and in global initializers. + * + * ## Wasm functions as JSFunctions + * + * Any exported function can be manipulated by JS and wasm code, and to both the + * exported function is represented as a JSFunction. To JS, that means that the + * function can be called in the same way as any other JSFunction. To Wasm, it + * means that the function is a reference with the same representation as + * externref. + * + * However, the JSFunction object is created only when the function value is + * actually exposed to JS the first time. The creation is performed by + * getExportedFunction(), below, as follows: + * + * - a function exported via the export section (or from asm.js) is created + * when the export object is created, which happens at instantiation time. + * + * - a function implicitly exported via a table is created when the table + * element is read (by JS or wasm) and a function value is needed to + * represent that value. Functions stored in tables by initializers have a + * special representation that does not require the function object to be + * created. + * + * - a function implicitly exported via a global initializer is created when + * the global is initialized. + * + * - a function referenced from a ref.func instruction in code is created when + * that instruction is executed the first time. + * + * The JSFunction representing a wasm function never changes: every reference to + * the wasm function that exposes the JSFunction gets the same JSFunction. In + * particular, imported functions already have a JSFunction representation (from + * JS or from their home module), and will be exposed using that representation. + * + * The mapping from a wasm function to its JSFunction is instance-specific, and + * held in a hashmap in the instance. If a module is shared across multiple + * instances, possibly in multiple threads, each instance will have its own + * JSFunction representing the wasm function. + * + * ## Stubs -- interpreter, eager, lazy, provisional, and absent + * + * While a Wasm exported function is just a JSFunction, the internal wasm ABI is + * neither the C++ ABI nor the JS JIT ABI, so there needs to be an extra step + * when C++ or JS JIT code calls wasm code. For this, execution passes through + * a stub that is adapted to both the JS caller and the wasm callee. + * + * ### Interpreter stubs and jit-entry stubs + * + * When JS interpreted code calls a wasm function, we end up in + * Instance::callExport() to execute the call. This function must enter wasm, + * and to do this it uses a stub that is specific to the wasm function (see + * GenerateInterpEntry) that is callable with the C++ interpreter ABI and which + * will convert arguments as necessary and enter compiled wasm code. + * + * The interpreter stub is created eagerly, when the module is compiled. + * + * However, the interpreter call path is slow, and when JS jitted code calls + * wasm we want to do better. In this case, there is a different, optimized + * stub that is to be invoked, and it uses the JIT ABI. This is the jit-entry + * stub for the function. Jitted code will call a wasm function's jit-entry + * stub to invoke the function with the JIT ABI. The stub will adapt the call + * to the wasm ABI. + * + * Some jit-entry stubs are created eagerly and some are created lazily. + * + * ### Eager jit-entry stubs + * + * The explicitly exported functions have stubs created for them eagerly. Eager + * stubs are created with their tier when the module is compiled, see + * ModuleGenerator::finishCodeTier(), which calls wasm::GenerateStubs(), which + * generates stubs for functions with eager stubs. + * + * An eager stub for tier-1 is upgraded to tier-2 if the module tiers up, see + * below. + * + * ### Lazy jit-entry stubs + * + * Stubs are created lazily for all implicitly exported functions. These + * functions may flow out to JS, but will only need a stub if they are ever + * called from jitted code. (That's true for explicitly exported functions too, + * but for them the presumption is that they will be called.) + * + * Lazy stubs are created only when they are needed, and they are /doubly/ lazy, + * see getExportedFunction(), below: A function implicitly exported via a table + * or global may be manipulated eagerly by host code without actually being + * called (maybe ever), so we do not generate a lazy stub when the function + * object escapes to JS, but instead delay stub generation until the function is + * actually called. + * + * ### The provisional lazy jit-entry stub + * + * However, JS baseline compilation needs to have a stub to start with in order + * to allow it to attach CacheIR data to the call (or it deoptimizes the call as + * a C++ call). Thus when the JSFunction for the wasm export is retrieved by JS + * code, a /provisional/ lazy jit-entry stub is associated with the function. + * The stub will invoke the wasm function on the slow interpreter path via + * callExport - if the function is ever called - and will cause a fast jit-entry + * stub to be created at the time of the call. The provisional lazy stub is + * shared globally, it contains no function-specific or context-specific data. + * + * Thus, the final lazy jit-entry stubs are eventually created by + * Instance::callExport, when a call is routed through it on the slow path for + * any of the reasons given above. + * + * ### Absent jit-entry stubs + * + * Some functions never get jit-entry stubs. The predicate canHaveJitEntry() + * determines if a wasm function gets a stub, and it will deny this if the + * function's signature exposes non-JS-compatible types (such as v128) or if + * stub optimization has been disabled by a jit option. Calls to these + * functions will continue to go via callExport and use the slow interpreter + * stub. + * + * ## The jit-entry jump table + * + * The mapping from the exported function to its jit-entry stub is implemented + * by the jit-entry jump table in the JumpTables object (see WasmCode.h). The + * jit-entry jump table entry for a function holds a stub that the jit can call + * to perform fast calls. + * + * While there is a single contiguous jump table, it has two logical sections: + * one for eager stubs, and one for lazy stubs. These sections are initialized + * and updated separately, using logic that is specific to each section. + * + * The value of the table element for an eager stub is a pointer to the stub + * code in the current tier. The pointer is installed just after the creation + * of the stub, before any code in the module is executed. If the module later + * tiers up, the eager jit-entry stub for tier-1 code is replaced by one for + * tier-2 code, see the next section. + * + * Initially the value of the jump table element for a lazy stub is null. + * + * If the function is retrieved by JS (by getExportedFunction()) and is not + * barred from having a jit-entry, then the stub is upgraded to the shared + * provisional lazy jit-entry stub. This upgrade happens to be racy if the + * module is shared, and so the update is atomic and only happens if the entry + * is already null. Since the provisional lazy stub is shared, this is fine; if + * several threads try to upgrade at the same time, it is to the same shared + * value. + * + * If the retrieved function is later invoked (via callExport()), the stub is + * upgraded to an actual jit-entry stub for the current code tier, again if the + * function is allowed to have a jit-entry. This is not racy -- though multiple + * threads can be trying to create a jit-entry stub at the same time, they do so + * under a lock and only the first to take the lock will be allowed to create a + * stub, the others will reuse the first-installed stub. + * + * If the module later tiers up, the lazy jit-entry stub for tier-1 code (if it + * exists) is replaced by one for tier-2 code, see the next section. + * + * (Note, the InterpEntry stub is never stored in the jit-entry table, as it + * uses the C++ ABI, not the JIT ABI. It is accessible through the + * FunctionEntry.) + * + * ### Interaction of the jit-entry jump table and tiering + * + * (For general info about tiering, see the comment in WasmCompile.cpp.) + * + * The jit-entry stub, whether eager or lazy, is specific to a code tier - a + * stub will invoke the code for its function for the tier. When we tier up, + * new jit-entry stubs must be created that reference tier-2 code, and must then + * be patched into the jit-entry table. The complication here is that, since + * the jump table is shared with its code between instances on multiple threads, + * tier-1 code is running on other threads and new tier-1 specific jit-entry + * stubs may be created concurrently with trying to create the tier-2 stubs on + * the thread that performs the tiering-up. Indeed, there may also be + * concurrent attempts to upgrade null jit-entries to the provisional lazy stub. + * + * Eager stubs: + * + * - Eager stubs for tier-2 code are patched in racily by Module::finishTier2() + * along with code pointers for tiering; nothing conflicts with these writes. + * + * Lazy stubs: + * + * - An upgrade from a null entry to a lazy provisional stub is atomic and can + * only happen if the entry is null, and it only happens in + * getExportedFunction(). No lazy provisional stub will be installed if + * there's another stub present. + * + * - The lazy tier-appropriate stub is installed by callExport() (really by + * EnsureEntryStubs()) during the first invocation of the exported function + * that reaches callExport(). That invocation must be from within JS, and so + * the jit-entry element can't be null, because a prior getExportedFunction() + * will have ensured that it is not: the lazy provisional stub will have been + * installed. Hence the installing of the lazy tier-appropriate stub does + * not race with the installing of the lazy provisional stub. + * + * - A lazy tier-1 stub is upgraded to a lazy tier-2 stub by + * Module::finishTier2(). The upgrade needs to ensure that all tier-1 stubs + * are upgraded, and that once the upgrade is finished, callExport() will + * only create tier-2 lazy stubs. (This upgrading does not upgrade lazy + * provisional stubs or absent stubs.) + * + * The locking protocol ensuring that all stubs are upgraded properly and + * that the system switches to creating tier-2 stubs is implemented in + * Module::finishTier2() and EnsureEntryStubs(): + * + * There are two locks, one per code tier. + * + * EnsureEntryStubs() is attempting to create a tier-appropriate lazy stub, + * so it takes the lock for the current best tier, checks to see if there is + * a stub, and exits if there is. If the tier changed racily it takes the + * other lock too, since that is now the lock for the best tier. Then it + * creates the stub, installs it, and releases the locks. Thus at most one + * stub per tier can be created at a time. + * + * Module::finishTier2() takes both locks (tier-1 before tier-2), thus + * preventing EnsureEntryStubs() from creating stubs while stub upgrading is + * going on, and itself waiting until EnsureEntryStubs() is not active. Once + * it has both locks, it upgrades all lazy stubs and makes tier-2 the new + * best tier. Should EnsureEntryStubs subsequently enter, it will find that + * a stub already exists at tier-2 and will exit early. + * + * (It would seem that the locking protocol could be simplified a little by + * having only one lock, hanging off the Code object, or by unconditionally + * taking both locks in EnsureEntryStubs(). However, in some cases where we + * acquire a lock the Code object is not readily available, so plumbing would + * have to be added, and in EnsureEntryStubs(), there are sometimes not two code + * tiers.) + * + * ## Stub lifetimes and serialization + * + * Eager jit-entry stub code, along with stub code for import functions, is + * serialized along with the tier-2 code for the module. + * + * Lazy stub code and thunks for builtin functions (including the provisional + * lazy jit-entry stub) are never serialized. + */ + +/* static */ +bool WasmInstanceObject::getExportedFunction( + JSContext* cx, Handle instanceObj, uint32_t funcIndex, + MutableHandleFunction fun) { + if (ExportMap::Ptr p = instanceObj->exports().lookup(funcIndex)) { + fun.set(p->value()); + return true; + } + + const Instance& instance = instanceObj->instance(); + const FuncExport& funcExport = + instance.metadata(instance.code().bestTier()).lookupFuncExport(funcIndex); + const FuncType& funcType = instance.metadata().getFuncExportType(funcExport); + unsigned numArgs = funcType.args().length(); + + if (instance.isAsmJS()) { + // asm.js needs to act like a normal JS function which means having the + // name from the original source and being callable as a constructor. + Rooted name(cx, instance.getFuncDisplayAtom(cx, funcIndex)); + if (!name) { + return false; + } + fun.set(NewNativeConstructor(cx, WasmCall, numArgs, name, + gc::AllocKind::FUNCTION_EXTENDED, + TenuredObject, FunctionFlags::ASMJS_CTOR)); + if (!fun) { + return false; + } + + // asm.js does not support jit entries. + fun->setWasmFuncIndex(funcIndex); + } else { + Rooted name(cx, NumberToAtom(cx, funcIndex)); + if (!name) { + return false; + } + RootedObject proto(cx); +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + proto = GlobalObject::getOrCreatePrototype(cx, JSProto_WasmFunction); + if (!proto) { + return false; + } +#endif + fun.set(NewFunctionWithProto( + cx, WasmCall, numArgs, FunctionFlags::WASM, nullptr, name, proto, + gc::AllocKind::FUNCTION_EXTENDED, TenuredObject)); + if (!fun) { + return false; + } + + // Some applications eagerly access all table elements which currently + // triggers worst-case behavior for lazy stubs, since each will allocate a + // separate 4kb code page. Most eagerly-accessed functions are not called, + // so use a shared, provisional (and slow) lazy stub as JitEntry and wait + // until Instance::callExport() to create the fast entry stubs. + if (funcType.canHaveJitEntry()) { + if (!funcExport.hasEagerStubs()) { + if (!EnsureBuiltinThunksInitialized()) { + return false; + } + void* provisionalLazyJitEntryStub = ProvisionalLazyJitEntryStub(); + MOZ_ASSERT(provisionalLazyJitEntryStub); + instance.code().setJitEntryIfNull(funcIndex, + provisionalLazyJitEntryStub); + } + fun->setWasmJitEntry(instance.code().getAddressOfJitEntry(funcIndex)); + } else { + fun->setWasmFuncIndex(funcIndex); + } + } + + fun->setExtendedSlot(FunctionExtended::WASM_INSTANCE_SLOT, + PrivateValue(const_cast(&instance))); + + const CodeTier& codeTier = + instance.code().codeTier(instance.code().bestTier()); + const CodeRange& codeRange = codeTier.metadata().codeRange(funcExport); + + fun->setExtendedSlot(FunctionExtended::WASM_FUNC_UNCHECKED_ENTRY_SLOT, + PrivateValue(codeTier.segment().base() + + codeRange.funcUncheckedCallEntry())); + + if (!instanceObj->exports().putNew(funcIndex, fun)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +const CodeRange& WasmInstanceObject::getExportedFunctionCodeRange( + JSFunction* fun, Tier tier) { + uint32_t funcIndex = ExportedFunctionToFuncIndex(fun); + MOZ_ASSERT(exports().lookup(funcIndex)->value() == fun); + const MetadataTier& metadata = instance().metadata(tier); + return metadata.codeRange(metadata.lookupFuncExport(funcIndex)); +} + +/* static */ +WasmInstanceScope* WasmInstanceObject::getScope( + JSContext* cx, Handle instanceObj) { + if (!instanceObj->getReservedSlot(INSTANCE_SCOPE_SLOT).isUndefined()) { + return (WasmInstanceScope*)instanceObj->getReservedSlot(INSTANCE_SCOPE_SLOT) + .toGCThing(); + } + + Rooted instanceScope( + cx, WasmInstanceScope::create(cx, instanceObj)); + if (!instanceScope) { + return nullptr; + } + + instanceObj->setReservedSlot(INSTANCE_SCOPE_SLOT, + PrivateGCThingValue(instanceScope)); + + return instanceScope; +} + +/* static */ +WasmFunctionScope* WasmInstanceObject::getFunctionScope( + JSContext* cx, Handle instanceObj, + uint32_t funcIndex) { + if (auto p = + instanceObj->scopes().asWasmFunctionScopeMap().lookup(funcIndex)) { + return p->value(); + } + + Rooted instanceScope( + cx, WasmInstanceObject::getScope(cx, instanceObj)); + if (!instanceScope) { + return nullptr; + } + + Rooted funcScope( + cx, WasmFunctionScope::create(cx, instanceScope, funcIndex)); + if (!funcScope) { + return nullptr; + } + + if (!instanceObj->scopes().asWasmFunctionScopeMap().putNew(funcIndex, + funcScope)) { + ReportOutOfMemory(cx); + return nullptr; + } + + return funcScope; +} + +bool wasm::IsWasmExportedFunction(JSFunction* fun) { + return fun->kind() == FunctionFlags::Wasm; +} + +Instance& wasm::ExportedFunctionToInstance(JSFunction* fun) { + return fun->wasmInstance(); +} + +WasmInstanceObject* wasm::ExportedFunctionToInstanceObject(JSFunction* fun) { + return fun->wasmInstance().object(); +} + +uint32_t wasm::ExportedFunctionToFuncIndex(JSFunction* fun) { + return fun->wasmInstance().code().getFuncIndex(fun); +} + +// ============================================================================ +// WebAssembly.Memory class and methods + +const JSClassOps WasmMemoryObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmMemoryObject::finalize, // finalize + nullptr, // call + nullptr, // construct + nullptr, // trace +}; + +const JSClass WasmMemoryObject::class_ = { + "WebAssembly.Memory", + JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_HAS_RESERVED_SLOTS(WasmMemoryObject::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &WasmMemoryObject::classOps_, &WasmMemoryObject::classSpec_}; + +const JSClass& WasmMemoryObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmMemoryName[] = "Memory"; + +static JSObject* CreateWasmMemoryPrototype(JSContext* cx, JSProtoKey key) { + RootedObject proto(cx, GlobalObject::createBlankPrototype( + cx, cx->global(), &WasmMemoryObject::protoClass_)); + if (!proto) { + return nullptr; + } + if (MemoryControlAvailable(cx)) { + if (!JS_DefineFunctions(cx, proto, + WasmMemoryObject::memoryControlMethods)) { + return nullptr; + } + } + return proto; +} + +const ClassSpec WasmMemoryObject::classSpec_ = { + CreateWasmConstructor, + CreateWasmMemoryPrototype, + WasmMemoryObject::static_methods, + nullptr, + WasmMemoryObject::methods, + WasmMemoryObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +/* static */ +void WasmMemoryObject::finalize(JS::GCContext* gcx, JSObject* obj) { + WasmMemoryObject& memory = obj->as(); + if (memory.hasObservers()) { + gcx->delete_(obj, &memory.observers(), MemoryUse::WasmMemoryObservers); + } +} + +/* static */ +WasmMemoryObject* WasmMemoryObject::create( + JSContext* cx, HandleArrayBufferObjectMaybeShared buffer, bool isHuge, + HandleObject proto) { + AutoSetNewObjectMetadata metadata(cx); + auto* obj = NewObjectWithGivenProto(cx, proto); + if (!obj) { + return nullptr; + } + + obj->initReservedSlot(BUFFER_SLOT, ObjectValue(*buffer)); + obj->initReservedSlot(ISHUGE_SLOT, BooleanValue(isHuge)); + MOZ_ASSERT(!obj->hasObservers()); + + return obj; +} + +/* static */ +bool WasmMemoryObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + if (!ThrowIfNotConstructing(cx, args, "Memory")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Memory", 1)) { + return false; + } + + if (!args.get(0).isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_DESC_ARG, "memory"); + return false; + } + + RootedObject obj(cx, &args[0].toObject()); + Limits limits; + if (!GetLimits(cx, obj, LimitsKind::Memory, &limits) || + !CheckLimits(cx, MaxMemoryLimitField(limits.indexType), + LimitsKind::Memory, &limits)) { + return false; + } + + if (Pages(limits.initial) > MaxMemoryPages(limits.indexType)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_MEM_IMP_LIMIT); + return false; + } + MemoryDesc memory(limits); + + RootedArrayBufferObjectMaybeShared buffer(cx); + if (!CreateWasmBuffer(cx, memory, &buffer)) { + return false; + } + + RootedObject proto(cx, + GetWasmConstructorPrototype(cx, args, JSProto_WasmMemory)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + Rooted memoryObj( + cx, WasmMemoryObject::create( + cx, buffer, IsHugeMemoryEnabled(limits.indexType), proto)); + if (!memoryObj) { + return false; + } + + args.rval().setObject(*memoryObj); + return true; +} + +static bool IsMemory(HandleValue v) { + return v.isObject() && v.toObject().is(); +} + +/* static */ +bool WasmMemoryObject::bufferGetterImpl(JSContext* cx, const CallArgs& args) { + Rooted memoryObj( + cx, &args.thisv().toObject().as()); + RootedArrayBufferObjectMaybeShared buffer(cx, &memoryObj->buffer()); + + if (memoryObj->isShared()) { + size_t memoryLength = memoryObj->volatileMemoryLength(); + MOZ_ASSERT(memoryLength >= buffer->byteLength()); + + if (memoryLength > buffer->byteLength()) { + RootedSharedArrayBufferObject newBuffer( + cx, SharedArrayBufferObject::New( + cx, memoryObj->sharedArrayRawBuffer(), memoryLength)); + if (!newBuffer) { + return false; + } + // OK to addReference after we try to allocate because the memoryObj + // keeps the rawBuffer alive. + if (!memoryObj->sharedArrayRawBuffer()->addReference()) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_SC_SAB_REFCNT_OFLO); + return false; + } + buffer = newBuffer; + memoryObj->setReservedSlot(BUFFER_SLOT, ObjectValue(*newBuffer)); + } + } + + args.rval().setObject(*buffer); + return true; +} + +/* static */ +bool WasmMemoryObject::bufferGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +const JSPropertySpec WasmMemoryObject::properties[] = { + JS_PSG("buffer", WasmMemoryObject::bufferGetter, JSPROP_ENUMERATE), + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Memory", JSPROP_READONLY), + JS_PS_END}; + +/* static */ +bool WasmMemoryObject::growImpl(JSContext* cx, const CallArgs& args) { + Rooted memory( + cx, &args.thisv().toObject().as()); + + if (!args.requireAtLeast(cx, "WebAssembly.Memory.grow", 1)) { + return false; + } + + uint32_t delta; + if (!EnforceRangeU32(cx, args.get(0), "Memory", "grow delta", &delta)) { + return false; + } + + uint32_t ret = grow(memory, delta, cx); + + if (ret == uint32_t(-1)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_GROW, + "memory"); + return false; + } + + args.rval().setInt32(int32_t(ret)); + return true; +} + +/* static */ +bool WasmMemoryObject::grow(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +/* static */ +bool WasmMemoryObject::discardImpl(JSContext* cx, const CallArgs& args) { + Rooted memory( + cx, &args.thisv().toObject().as()); + + if (!args.requireAtLeast(cx, "WebAssembly.Memory.discard", 2)) { + return false; + } + + uint64_t byteOffset; + if (!EnforceRangeU64(cx, args.get(0), "Memory", "byte offset", &byteOffset)) { + return false; + } + + uint64_t byteLen; + if (!EnforceRangeU64(cx, args.get(1), "Memory", "length", &byteLen)) { + return false; + } + + if (byteOffset % wasm::PageSize != 0 || byteLen % wasm::PageSize != 0) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_UNALIGNED_ACCESS); + return false; + } + + if (!wasm::MemoryBoundsCheck(byteOffset, byteLen, + memory->volatileMemoryLength())) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_OUT_OF_BOUNDS); + return false; + } + + discard(memory, byteOffset, byteLen, cx); + + args.rval().setUndefined(); + return true; +} + +/* static */ +bool WasmMemoryObject::discard(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +const JSFunctionSpec WasmMemoryObject::methods[] = { +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + JS_FN("type", WasmMemoryObject::type, 0, JSPROP_ENUMERATE), +#endif + JS_FN("grow", WasmMemoryObject::grow, 1, JSPROP_ENUMERATE), JS_FS_END}; + +const JSFunctionSpec WasmMemoryObject::memoryControlMethods[] = { + JS_FN("discard", WasmMemoryObject::discard, 2, JSPROP_ENUMERATE), + JS_FS_END}; + +const JSFunctionSpec WasmMemoryObject::static_methods[] = {JS_FS_END}; + +ArrayBufferObjectMaybeShared& WasmMemoryObject::buffer() const { + return getReservedSlot(BUFFER_SLOT) + .toObject() + .as(); +} + +WasmSharedArrayRawBuffer* WasmMemoryObject::sharedArrayRawBuffer() const { + MOZ_ASSERT(isShared()); + return buffer().as().rawWasmBufferObject(); +} + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS +bool WasmMemoryObject::typeImpl(JSContext* cx, const CallArgs& args) { + Rooted memoryObj( + cx, &args.thisv().toObject().as()); + RootedObject typeObj( + cx, MemoryTypeToObject(cx, memoryObj->isShared(), memoryObj->indexType(), + memoryObj->volatilePages(), + memoryObj->sourceMaxPages())); + if (!typeObj) { + return false; + } + args.rval().setObject(*typeObj); + return true; +} + +bool WasmMemoryObject::type(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} +#endif + +size_t WasmMemoryObject::volatileMemoryLength() const { + if (isShared()) { + return sharedArrayRawBuffer()->volatileByteLength(); + } + return buffer().byteLength(); +} + +wasm::Pages WasmMemoryObject::volatilePages() const { + if (isShared()) { + return sharedArrayRawBuffer()->volatileWasmPages(); + } + return buffer().wasmPages(); +} + +wasm::Pages WasmMemoryObject::clampedMaxPages() const { + if (isShared()) { + return sharedArrayRawBuffer()->wasmClampedMaxPages(); + } + return buffer().wasmClampedMaxPages(); +} + +Maybe WasmMemoryObject::sourceMaxPages() const { + if (isShared()) { + return Some(sharedArrayRawBuffer()->wasmSourceMaxPages()); + } + return buffer().wasmSourceMaxPages(); +} + +wasm::IndexType WasmMemoryObject::indexType() const { + if (isShared()) { + return sharedArrayRawBuffer()->wasmIndexType(); + } + return buffer().wasmIndexType(); +} + +bool WasmMemoryObject::isShared() const { + return buffer().is(); +} + +bool WasmMemoryObject::hasObservers() const { + return !getReservedSlot(OBSERVERS_SLOT).isUndefined(); +} + +WasmMemoryObject::InstanceSet& WasmMemoryObject::observers() const { + MOZ_ASSERT(hasObservers()); + return *reinterpret_cast( + getReservedSlot(OBSERVERS_SLOT).toPrivate()); +} + +WasmMemoryObject::InstanceSet* WasmMemoryObject::getOrCreateObservers( + JSContext* cx) { + if (!hasObservers()) { + auto observers = MakeUnique(cx->zone(), cx->zone()); + if (!observers) { + ReportOutOfMemory(cx); + return nullptr; + } + + InitReservedSlot(this, OBSERVERS_SLOT, observers.release(), + MemoryUse::WasmMemoryObservers); + } + + return &observers(); +} + +bool WasmMemoryObject::isHuge() const { + return getReservedSlot(ISHUGE_SLOT).toBoolean(); +} + +bool WasmMemoryObject::movingGrowable() const { + return !isHuge() && !buffer().wasmSourceMaxPages(); +} + +size_t WasmMemoryObject::boundsCheckLimit() const { + if (!buffer().isWasm() || isHuge()) { + return buffer().byteLength(); + } + size_t mappedSize = buffer().wasmMappedSize(); +#if !defined(JS_64BIT) + // See clamping performed in CreateSpecificWasmBuffer(). On 32-bit systems + // we do not want to overflow a uint32_t. For the other 64-bit compilers, + // all constraints are implied by the largest accepted value for a memory's + // max field. + MOZ_ASSERT(mappedSize < UINT32_MAX); +#endif + MOZ_ASSERT(mappedSize % wasm::PageSize == 0); + MOZ_ASSERT(mappedSize >= wasm::GuardSize); + MOZ_ASSERT(wasm::IsValidBoundsCheckImmediate(mappedSize - wasm::GuardSize)); + size_t limit = mappedSize - wasm::GuardSize; + MOZ_ASSERT(limit <= MaxMemoryBoundsCheckLimit(indexType())); + return limit; +} + +bool WasmMemoryObject::addMovingGrowObserver(JSContext* cx, + WasmInstanceObject* instance) { + MOZ_ASSERT(movingGrowable()); + + InstanceSet* observers = getOrCreateObservers(cx); + if (!observers) { + return false; + } + + if (!observers->putNew(instance)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +/* static */ +uint64_t WasmMemoryObject::growShared(Handle memory, + uint64_t delta) { + WasmSharedArrayRawBuffer* rawBuf = memory->sharedArrayRawBuffer(); + WasmSharedArrayRawBuffer::Lock lock(rawBuf); + + Pages oldNumPages = rawBuf->volatileWasmPages(); + Pages newPages = oldNumPages; + if (!newPages.checkedIncrement(Pages(delta))) { + return uint64_t(int64_t(-1)); + } + + if (!rawBuf->wasmGrowToPagesInPlace(lock, memory->indexType(), newPages)) { + return uint64_t(int64_t(-1)); + } + // New buffer objects will be created lazily in all agents (including in + // this agent) by bufferGetterImpl, above, so no more work to do here. + + return oldNumPages.value(); +} + +/* static */ +uint64_t WasmMemoryObject::grow(Handle memory, + uint64_t delta, JSContext* cx) { + if (memory->isShared()) { + return growShared(memory, delta); + } + + RootedArrayBufferObject oldBuf(cx, &memory->buffer().as()); + +#if !defined(JS_64BIT) + // TODO (large ArrayBuffer): See more information at the definition of + // MaxMemoryBytes(). + MOZ_ASSERT(MaxMemoryBytes(memory->indexType()) <= UINT32_MAX, + "Avoid 32-bit overflows"); +#endif + + Pages oldNumPages = oldBuf->wasmPages(); + Pages newPages = oldNumPages; + if (!newPages.checkedIncrement(Pages(delta))) { + return uint64_t(int64_t(-1)); + } + + RootedArrayBufferObject newBuf(cx); + + if (memory->movingGrowable()) { + MOZ_ASSERT(!memory->isHuge()); + if (!ArrayBufferObject::wasmMovingGrowToPages(memory->indexType(), newPages, + oldBuf, &newBuf, cx)) { + return uint64_t(int64_t(-1)); + } + } else if (!ArrayBufferObject::wasmGrowToPagesInPlace( + memory->indexType(), newPages, oldBuf, &newBuf, cx)) { + return uint64_t(int64_t(-1)); + } + + memory->setReservedSlot(BUFFER_SLOT, ObjectValue(*newBuf)); + + // Only notify moving-grow-observers after the BUFFER_SLOT has been updated + // since observers will call buffer(). + if (memory->hasObservers()) { + for (InstanceSet::Range r = memory->observers().all(); !r.empty(); + r.popFront()) { + r.front()->instance().onMovingGrowMemory(); + } + } + + return oldNumPages.value(); +} + +/* static */ +void WasmMemoryObject::discard(Handle memory, + uint64_t byteOffset, uint64_t byteLen, + JSContext* cx) { + if (memory->isShared()) { + RootedSharedArrayBufferObject buf( + cx, &memory->buffer().as()); + SharedArrayBufferObject::wasmDiscard(buf, byteOffset, byteLen); + } else { + RootedArrayBufferObject buf(cx, &memory->buffer().as()); + ArrayBufferObject::wasmDiscard(buf, byteOffset, byteLen); + } +} + +bool js::wasm::IsSharedWasmMemoryObject(JSObject* obj) { + WasmMemoryObject* mobj = obj->maybeUnwrapIf(); + return mobj && mobj->isShared(); +} + +// ============================================================================ +// WebAssembly.Table class and methods + +const JSClassOps WasmTableObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmTableObject::finalize, // finalize + nullptr, // call + nullptr, // construct + WasmTableObject::trace, // trace +}; + +const JSClass WasmTableObject::class_ = { + "WebAssembly.Table", + JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_HAS_RESERVED_SLOTS(WasmTableObject::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &WasmTableObject::classOps_, &WasmTableObject::classSpec_}; + +const JSClass& WasmTableObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmTableName[] = "Table"; + +const ClassSpec WasmTableObject::classSpec_ = { + CreateWasmConstructor, + GenericCreatePrototype, + WasmTableObject::static_methods, + nullptr, + WasmTableObject::methods, + WasmTableObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +bool WasmTableObject::isNewborn() const { + MOZ_ASSERT(is()); + return getReservedSlot(TABLE_SLOT).isUndefined(); +} + +/* static */ +void WasmTableObject::finalize(JS::GCContext* gcx, JSObject* obj) { + WasmTableObject& tableObj = obj->as(); + if (!tableObj.isNewborn()) { + auto& table = tableObj.table(); + gcx->release(obj, &table, table.gcMallocBytes(), MemoryUse::WasmTableTable); + } +} + +/* static */ +void WasmTableObject::trace(JSTracer* trc, JSObject* obj) { + WasmTableObject& tableObj = obj->as(); + if (!tableObj.isNewborn()) { + tableObj.table().tracePrivate(trc); + } +} + +// Return the JS value to use when a parameter to a function requiring a table +// value is omitted. An implementation of [1]. +// +// [1] +// https://webassembly.github.io/reference-types/js-api/index.html#defaultvalue +static Value RefTypeDefautValue(wasm::RefType tableType) { + return tableType.isExtern() ? UndefinedValue() : NullValue(); +} + +static bool CheckRefTypeValue(JSContext* cx, wasm::RefType type, + HandleValue value) { + RootedFunction fun(cx); + RootedAnyRef any(cx, AnyRef::null()); + + return CheckRefType(cx, type, value, &fun, &any); +} + +/* static */ +WasmTableObject* WasmTableObject::create(JSContext* cx, uint32_t initialLength, + Maybe maximumLength, + wasm::RefType tableType, + HandleObject proto) { + AutoSetNewObjectMetadata metadata(cx); + Rooted obj( + cx, NewObjectWithGivenProto(cx, proto)); + if (!obj) { + return nullptr; + } + + MOZ_ASSERT(obj->isNewborn()); + + TableDesc td(tableType, initialLength, maximumLength, Nothing(), + /*isAsmJS*/ false, + /*isImported=*/true, /*isExported=*/true); + + SharedTable table = Table::create(cx, td, obj); + if (!table) { + return nullptr; + } + + size_t size = table->gcMallocBytes(); + InitReservedSlot(obj, TABLE_SLOT, table.forget().take(), size, + MemoryUse::WasmTableTable); + + MOZ_ASSERT(!obj->isNewborn()); + return obj; +} + +/* static */ +bool WasmTableObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + if (!ThrowIfNotConstructing(cx, args, "Table")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Table", 1)) { + return false; + } + + if (!args.get(0).isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_DESC_ARG, "table"); + return false; + } + + RootedObject obj(cx, &args[0].toObject()); + + JSAtom* elementAtom = Atomize(cx, "element", strlen("element")); + if (!elementAtom) { + return false; + } + RootedId elementId(cx, AtomToId(elementAtom)); + + RootedValue elementVal(cx); + if (!GetProperty(cx, obj, obj, elementId, &elementVal)) { + return false; + } + + RefType tableType; + if (!ToRefType(cx, elementVal, &tableType)) { + return false; + } + + Limits limits; + if (!GetLimits(cx, obj, LimitsKind::Table, &limits) || + !CheckLimits(cx, MaxTableLimitField, LimitsKind::Table, &limits)) { + return false; + } + + // Converting limits for a table only supports i32 + MOZ_ASSERT(limits.indexType == IndexType::I32); + + if (limits.initial > MaxTableLength) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_TABLE_IMP_LIMIT); + return false; + } + + RootedObject proto(cx, + GetWasmConstructorPrototype(cx, args, JSProto_WasmTable)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + // The rest of the runtime expects table limits to be within a 32-bit range. + static_assert(MaxTableLimitField <= UINT32_MAX, "invariant"); + uint32_t initialLength = uint32_t(limits.initial); + Maybe maximumLength; + if (limits.maximum) { + maximumLength = Some(uint32_t(*limits.maximum)); + } + + Rooted table( + cx, WasmTableObject::create(cx, initialLength, maximumLength, tableType, + proto)); + if (!table) { + return false; + } + + // Initialize the table to a default value + RootedValue initValue( + cx, args.length() < 2 ? RefTypeDefautValue(tableType) : args[1]); + if (!CheckRefTypeValue(cx, tableType, initValue)) { + return false; + } + + // Skip initializing the table if the fill value is null, as that is the + // default value. + if (!initValue.isNull() && + !table->fillRange(cx, 0, initialLength, initValue)) { + return false; + } +#ifdef DEBUG + // Assert that null is the default value of a new table. + if (initValue.isNull()) { + table->table().assertRangeNull(0, initialLength); + } + if (!tableType.isNullable()) { + table->table().assertRangeNotNull(0, initialLength); + } +#endif + + args.rval().setObject(*table); + return true; +} + +static bool IsTable(HandleValue v) { + return v.isObject() && v.toObject().is(); +} + +/* static */ +bool WasmTableObject::lengthGetterImpl(JSContext* cx, const CallArgs& args) { + args.rval().setNumber( + args.thisv().toObject().as().table().length()); + return true; +} + +/* static */ +bool WasmTableObject::lengthGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +const JSPropertySpec WasmTableObject::properties[] = { + JS_PSG("length", WasmTableObject::lengthGetter, JSPROP_ENUMERATE), + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Table", JSPROP_READONLY), + JS_PS_END}; + +static bool ToTableIndex(JSContext* cx, HandleValue v, const Table& table, + const char* noun, uint32_t* index) { + if (!EnforceRangeU32(cx, v, "Table", noun, index)) { + return false; + } + + if (*index >= table.length()) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_RANGE, "Table", noun); + return false; + } + + return true; +} + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS +/* static */ +bool WasmTableObject::typeImpl(JSContext* cx, const CallArgs& args) { + Table& table = args.thisv().toObject().as().table(); + RootedObject typeObj(cx, TableTypeToObject(cx, table.elemType(), + table.length(), table.maximum())); + if (!typeObj) { + return false; + } + args.rval().setObject(*typeObj); + return true; +} + +/* static */ +bool WasmTableObject::type(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} +#endif + +/* static */ +bool WasmTableObject::getImpl(JSContext* cx, const CallArgs& args) { + Rooted tableObj( + cx, &args.thisv().toObject().as()); + const Table& table = tableObj->table(); + + if (!args.requireAtLeast(cx, "WebAssembly.Table.get", 1)) { + return false; + } + + uint32_t index; + if (!ToTableIndex(cx, args.get(0), table, "get index", &index)) { + return false; + } + + return table.getValue(cx, index, args.rval()); +} + +/* static */ +bool WasmTableObject::get(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +/* static */ +bool WasmTableObject::setImpl(JSContext* cx, const CallArgs& args) { + Rooted tableObj( + cx, &args.thisv().toObject().as()); + Table& table = tableObj->table(); + + if (!args.requireAtLeast(cx, "WebAssembly.Table.set", 1)) { + return false; + } + + uint32_t index; + if (!ToTableIndex(cx, args.get(0), table, "set index", &index)) { + return false; + } + + RootedValue fillValue( + cx, args.length() < 2 ? RefTypeDefautValue(table.elemType()) : args[1]); + if (!tableObj->fillRange(cx, index, 1, fillValue)) { + return false; + } + + args.rval().setUndefined(); + return true; +} + +/* static */ +bool WasmTableObject::set(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +/* static */ +bool WasmTableObject::growImpl(JSContext* cx, const CallArgs& args) { + Rooted tableObj( + cx, &args.thisv().toObject().as()); + Table& table = tableObj->table(); + + if (!args.requireAtLeast(cx, "WebAssembly.Table.grow", 1)) { + return false; + } + + uint32_t delta; + if (!EnforceRangeU32(cx, args.get(0), "Table", "grow delta", &delta)) { + return false; + } + + RootedValue fillValue( + cx, args.length() < 2 ? RefTypeDefautValue(table.elemType()) : args[1]); + if (!CheckRefTypeValue(cx, table.elemType(), fillValue)) { + return false; + } + + uint32_t oldLength = table.grow(delta); + + if (oldLength == uint32_t(-1)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_GROW, + "table"); + return false; + } + + // Skip filling the grown range of the table if the fill value is null, as + // that is the default value. + if (!fillValue.isNull() && + !tableObj->fillRange(cx, oldLength, delta, fillValue)) { + return false; + } +#ifdef DEBUG + // Assert that null is the default value of the grown range. + if (fillValue.isNull()) { + table.assertRangeNull(oldLength, delta); + } + if (!table.elemType().isNullable()) { + table.assertRangeNotNull(oldLength, delta); + } +#endif + + args.rval().setInt32(int32_t(oldLength)); + return true; +} + +/* static */ +bool WasmTableObject::grow(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +const JSFunctionSpec WasmTableObject::methods[] = { +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + JS_FN("type", WasmTableObject::type, 0, JSPROP_ENUMERATE), +#endif + JS_FN("get", WasmTableObject::get, 1, JSPROP_ENUMERATE), + JS_FN("set", WasmTableObject::set, 2, JSPROP_ENUMERATE), + JS_FN("grow", WasmTableObject::grow, 1, JSPROP_ENUMERATE), JS_FS_END}; + +const JSFunctionSpec WasmTableObject::static_methods[] = {JS_FS_END}; + +Table& WasmTableObject::table() const { + return *(Table*)getReservedSlot(TABLE_SLOT).toPrivate(); +} + +bool WasmTableObject::fillRange(JSContext* cx, uint32_t index, uint32_t length, + HandleValue value) const { + Table& tab = table(); + + // All consumers are required to either bounds check or statically be in + // bounds + MOZ_ASSERT(uint64_t(index) + uint64_t(length) <= tab.length()); + + RootedFunction fun(cx); + RootedAnyRef any(cx, AnyRef::null()); + if (!CheckRefType(cx, tab.elemType(), value, &fun, &any)) { + return false; + } + switch (tab.repr()) { + case TableRepr::Func: + MOZ_RELEASE_ASSERT(!tab.isAsmJS()); + tab.fillFuncRef(index, length, FuncRef::fromJSFunction(fun), cx); + break; + case TableRepr::Ref: + tab.fillAnyRef(index, length, any); + break; + } + return true; +} + +// ============================================================================ +// WebAssembly.global class and methods + +const JSClassOps WasmGlobalObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmGlobalObject::finalize, // finalize + nullptr, // call + nullptr, // construct + WasmGlobalObject::trace, // trace +}; + +const JSClass WasmGlobalObject::class_ = { + "WebAssembly.Global", + JSCLASS_HAS_RESERVED_SLOTS(WasmGlobalObject::RESERVED_SLOTS) | + JSCLASS_BACKGROUND_FINALIZE, + &WasmGlobalObject::classOps_, &WasmGlobalObject::classSpec_}; + +const JSClass& WasmGlobalObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmGlobalName[] = "Global"; + +const ClassSpec WasmGlobalObject::classSpec_ = { + CreateWasmConstructor, + GenericCreatePrototype, + WasmGlobalObject::static_methods, + nullptr, + WasmGlobalObject::methods, + WasmGlobalObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +/* static */ +void WasmGlobalObject::trace(JSTracer* trc, JSObject* obj) { + WasmGlobalObject* global = reinterpret_cast(obj); + if (global->isNewborn()) { + // This can happen while we're allocating the object, in which case + // every single slot of the object is not defined yet. In particular, + // there's nothing to trace yet. + return; + } + global->val().get().trace(trc); +} + +/* static */ +void WasmGlobalObject::finalize(JS::GCContext* gcx, JSObject* obj) { + WasmGlobalObject* global = reinterpret_cast(obj); + if (!global->isNewborn()) { + gcx->delete_(obj, &global->val(), MemoryUse::WasmGlobalCell); + } +} + +/* static */ +WasmGlobalObject* WasmGlobalObject::create(JSContext* cx, HandleVal value, + bool isMutable, HandleObject proto) { + Rooted obj( + cx, NewObjectWithGivenProto(cx, proto)); + if (!obj) { + return nullptr; + } + + MOZ_ASSERT(obj->isNewborn()); + MOZ_ASSERT(obj->isTenured(), "assumed by global.set post barriers"); + + GCPtrVal* val = js_new(Val()); + if (!val) { + ReportOutOfMemory(cx); + return nullptr; + } + obj->initReservedSlot(MUTABLE_SLOT, JS::BooleanValue(isMutable)); + InitReservedSlot(obj, VAL_SLOT, val, MemoryUse::WasmGlobalCell); + + // It's simpler to initialize the cell after the object has been created, + // to avoid needing to root the cell before the object creation. + obj->val() = value.get(); + + MOZ_ASSERT(!obj->isNewborn()); + + return obj; +} + +/* static */ +bool WasmGlobalObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + if (!ThrowIfNotConstructing(cx, args, "Global")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Global", 1)) { + return false; + } + + if (!args.get(0).isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_DESC_ARG, "global"); + return false; + } + + RootedObject obj(cx, &args[0].toObject()); + + // Extract properties in lexicographic order per spec. + + RootedValue mutableVal(cx); + if (!JS_GetProperty(cx, obj, "mutable", &mutableVal)) { + return false; + } + + RootedValue typeVal(cx); + if (!JS_GetProperty(cx, obj, "value", &typeVal)) { + return false; + } + + ValType globalType; + if (!ToValType(cx, typeVal, &globalType)) { + return false; + } + + if (!globalType.isExposable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + + bool isMutable = ToBoolean(mutableVal); + + // Extract the initial value, or provide a suitable default. + RootedVal globalVal(cx, globalType); + + // Override with non-undefined value, if provided. + RootedValue valueVal(cx); + if (globalType.isRefType()) { + valueVal.set(args.length() < 2 ? RefTypeDefautValue(globalType.refType()) + : args[1]); + if (!Val::fromJSValue(cx, globalType, valueVal, &globalVal)) { + return false; + } + } else { + valueVal.set(args.get(1)); + if (!valueVal.isUndefined() && + !Val::fromJSValue(cx, globalType, valueVal, &globalVal)) { + return false; + } + } + + RootedObject proto(cx, + GetWasmConstructorPrototype(cx, args, JSProto_WasmGlobal)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + WasmGlobalObject* global = + WasmGlobalObject::create(cx, globalVal, isMutable, proto); + if (!global) { + return false; + } + + args.rval().setObject(*global); + return true; +} + +static bool IsGlobal(HandleValue v) { + return v.isObject() && v.toObject().is(); +} + +/* static */ +bool WasmGlobalObject::valueGetterImpl(JSContext* cx, const CallArgs& args) { + const WasmGlobalObject& globalObj = + args.thisv().toObject().as(); + if (!globalObj.type().isExposable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + return globalObj.val().get().toJSValue(cx, args.rval()); +} + +/* static */ +bool WasmGlobalObject::valueGetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +/* static */ +bool WasmGlobalObject::valueSetterImpl(JSContext* cx, const CallArgs& args) { + if (!args.requireAtLeast(cx, "WebAssembly.Global setter", 1)) { + return false; + } + + Rooted global( + cx, &args.thisv().toObject().as()); + if (!global->isMutable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_GLOBAL_IMMUTABLE); + return false; + } + + RootedVal val(cx); + if (!Val::fromJSValue(cx, global->type(), args.get(0), &val)) { + return false; + } + global->val() = val.get(); + + args.rval().setUndefined(); + return true; +} + +/* static */ +bool WasmGlobalObject::valueSetter(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +const JSPropertySpec WasmGlobalObject::properties[] = { + JS_PSGS("value", WasmGlobalObject::valueGetter, + WasmGlobalObject::valueSetter, JSPROP_ENUMERATE), + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Global", JSPROP_READONLY), + JS_PS_END}; + +const JSFunctionSpec WasmGlobalObject::methods[] = { +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + JS_FN("type", WasmGlobalObject::type, 0, JSPROP_ENUMERATE), +#endif + JS_FN(js_valueOf_str, WasmGlobalObject::valueGetter, 0, JSPROP_ENUMERATE), + JS_FS_END}; + +const JSFunctionSpec WasmGlobalObject::static_methods[] = {JS_FS_END}; + +bool WasmGlobalObject::isMutable() const { + return getReservedSlot(MUTABLE_SLOT).toBoolean(); +} + +ValType WasmGlobalObject::type() const { return val().get().type(); } + +GCPtrVal& WasmGlobalObject::val() const { + return *reinterpret_cast(getReservedSlot(VAL_SLOT).toPrivate()); +} + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS +/* static */ +bool WasmGlobalObject::typeImpl(JSContext* cx, const CallArgs& args) { + Rooted global( + cx, &args.thisv().toObject().as()); + RootedObject typeObj( + cx, GlobalTypeToObject(cx, global->type(), global->isMutable())); + if (!typeObj) { + return false; + } + args.rval().setObject(*typeObj); + return true; +} + +/* static */ +bool WasmGlobalObject::type(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} +#endif + +// ============================================================================ +// WebAssembly.Tag class and methods + +const JSClassOps WasmTagObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmTagObject::finalize, // finalize + nullptr, // call + nullptr, // construct + nullptr, // trace +}; + +const JSClass WasmTagObject::class_ = { + "WebAssembly.Tag", + JSCLASS_HAS_RESERVED_SLOTS(WasmTagObject::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &WasmTagObject::classOps_, &WasmTagObject::classSpec_}; + +const JSClass& WasmTagObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmTagName[] = "Tag"; + +const ClassSpec WasmTagObject::classSpec_ = { + CreateWasmConstructor, + GenericCreatePrototype, + WasmTagObject::static_methods, + nullptr, + WasmTagObject::methods, + WasmTagObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +/* static */ +void WasmTagObject::finalize(JS::GCContext* gcx, JSObject* obj) { + WasmTagObject& tagObj = obj->as(); + tagObj.tagType()->Release(); +} + +static bool IsTag(HandleValue v) { + return v.isObject() && v.toObject().is(); +} + +bool WasmTagObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + if (!ThrowIfNotConstructing(cx, args, "Tag")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Tag", 1)) { + return false; + } + + if (!args.get(0).isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_DESC_ARG, "tag"); + return false; + } + + RootedObject obj(cx, &args[0].toObject()); + RootedValue paramsVal(cx); + if (!JS_GetProperty(cx, obj, "parameters", ¶msVal)) { + return false; + } + + ValTypeVector params; + if (!ParseValTypes(cx, paramsVal, params)) { + return false; + } + wasm::MutableTagType tagType = js_new(); + if (!tagType || !tagType->initialize(std::move(params))) { + return false; + } + + RootedObject proto(cx, + GetWasmConstructorPrototype(cx, args, JSProto_WasmTag)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + Rooted tagObj(cx, WasmTagObject::create(cx, tagType, proto)); + if (!tagObj) { + return false; + } + + args.rval().setObject(*tagObj); + return true; +} + +/* static */ +WasmTagObject* WasmTagObject::create(JSContext* cx, + const wasm::SharedTagType& tagType, + HandleObject proto) { + Rooted obj(cx, + NewObjectWithGivenProto(cx, proto)); + if (!obj) { + return nullptr; + } + + tagType.get()->AddRef(); + obj->initReservedSlot(TYPE_SLOT, PrivateValue((void*)tagType.get())); + + return obj; +} + +const JSPropertySpec WasmTagObject::properties[] = { + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Tag", JSPROP_READONLY), + JS_PS_END}; + +#ifdef ENABLE_WASM_TYPE_REFLECTIONS +/* static */ +bool WasmTagObject::typeImpl(JSContext* cx, const CallArgs& args) { + Rooted tag(cx, &args.thisv().toObject().as()); + RootedObject typeObj(cx, TagTypeToObject(cx, tag->valueTypes())); + if (!typeObj) { + return false; + } + args.rval().setObject(*typeObj); + return true; +} + +/* static */ +bool WasmTagObject::type(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} +#endif + +const JSFunctionSpec WasmTagObject::methods[] = { +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + JS_FN("type", WasmTagObject::type, 0, JSPROP_ENUMERATE), +#endif + JS_FS_END}; + +const JSFunctionSpec WasmTagObject::static_methods[] = {JS_FS_END}; + +const TagType* WasmTagObject::tagType() const { + return (const TagType*)getFixedSlot(TYPE_SLOT).toPrivate(); +}; + +const wasm::ValTypeVector& WasmTagObject::valueTypes() const { + return tagType()->argTypes_; +}; + +wasm::ResultType WasmTagObject::resultType() const { + return wasm::ResultType::Vector(valueTypes()); +} + +// ============================================================================ +// WebAssembly.Exception class and methods + +const JSClassOps WasmExceptionObject::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + WasmExceptionObject::finalize, // finalize + nullptr, // call + nullptr, // construct + WasmExceptionObject::trace, // trace +}; + +const JSClass WasmExceptionObject::class_ = { + "WebAssembly.Exception", + JSCLASS_HAS_RESERVED_SLOTS(WasmExceptionObject::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &WasmExceptionObject::classOps_, &WasmExceptionObject::classSpec_}; + +const JSClass& WasmExceptionObject::protoClass_ = PlainObject::class_; + +static constexpr char WasmExceptionName[] = "Exception"; + +const ClassSpec WasmExceptionObject::classSpec_ = { + CreateWasmConstructor, + GenericCreatePrototype, + WasmExceptionObject::static_methods, + nullptr, + WasmExceptionObject::methods, + WasmExceptionObject::properties, + nullptr, + ClassSpec::DontDefineConstructor}; + +/* static */ +void WasmExceptionObject::finalize(JS::GCContext* gcx, JSObject* obj) { + WasmExceptionObject& exnObj = obj->as(); + if (exnObj.isNewborn()) { + return; + } + gcx->free_(obj, exnObj.typedMem(), exnObj.tagType()->size_, + MemoryUse::WasmExceptionData); + exnObj.tagType()->Release(); +} + +/* static */ +void WasmExceptionObject::trace(JSTracer* trc, JSObject* obj) { + WasmExceptionObject& exnObj = obj->as(); + if (exnObj.isNewborn()) { + return; + } + + wasm::SharedTagType tag = exnObj.tagType(); + const wasm::ValTypeVector& params = tag->argTypes_; + const wasm::TagOffsetVector& offsets = tag->argOffsets_; + uint8_t* typedMem = exnObj.typedMem(); + for (size_t i = 0; i < params.length(); i++) { + ValType paramType = params[i]; + if (paramType.isRefRepr()) { + GCPtr* objectPtr = + reinterpret_cast*>(typedMem + offsets[i]); + TraceNullableEdge(trc, objectPtr, "reference-obj"); + } + } +} + +static bool IsException(HandleValue v) { + return v.isObject() && v.toObject().is(); +} + +struct ExceptionOptions { + bool traceStack; + + ExceptionOptions() : traceStack(false) {} + + [[nodiscard]] bool init(JSContext* cx, HandleValue val); +}; + +bool ExceptionOptions::init(JSContext* cx, HandleValue val) { + if (val.isNullOrUndefined()) { + return true; + } + + if (!val.isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_OPTIONS); + return false; + } + RootedObject obj(cx, &val.toObject()); + + // Get `traceStack` and coerce to boolean + RootedValue traceStackVal(cx); + if (!JS_GetProperty(cx, obj, "traceStack", &traceStackVal)) { + return false; + } + traceStack = ToBoolean(traceStackVal); + + return true; +} + +bool WasmExceptionObject::construct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + if (!ThrowIfNotConstructing(cx, args, "Exception")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Exception", 2)) { + return false; + } + + if (!IsTag(args[0])) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_ARG); + return false; + } + Rooted exnTag(cx, &args[0].toObject().as()); + + if (!args.get(1).isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_PAYLOAD); + return false; + } + + JS::ForOfIterator iterator(cx); + if (!iterator.init(args.get(1), JS::ForOfIterator::ThrowOnNonIterable)) { + return false; + } + + // Get the optional 'options' parameter + ExceptionOptions options; + if (!options.init(cx, args.get(2))) { + return false; + } + + // Trace the stack if requested + RootedObject stack(cx); + if (options.traceStack && !CaptureStack(cx, &stack)) { + return false; + } + + RootedObject proto( + cx, GetWasmConstructorPrototype(cx, args, JSProto_WasmException)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + Rooted exnObj( + cx, WasmExceptionObject::create(cx, exnTag, stack, proto)); + if (!exnObj) { + return false; + } + + wasm::SharedTagType tagType = exnObj->tagType(); + const wasm::ValTypeVector& params = tagType->argTypes_; + const wasm::TagOffsetVector& offsets = tagType->argOffsets_; + + RootedValue nextArg(cx); + for (size_t i = 0; i < params.length(); i++) { + bool done; + if (!iterator.next(&nextArg, &done)) { + return false; + } + if (done) { + UniqueChars expected(JS_smprintf("%zu", params.length())); + UniqueChars got(JS_smprintf("%zu", i)); + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_PAYLOAD_LEN, expected.get(), + got.get()); + return false; + } + + if (!exnObj->initValue(cx, offsets[i], params[i], nextArg)) { + return false; + } + } + + args.rval().setObject(*exnObj); + return true; +} + +/* static */ +WasmExceptionObject* WasmExceptionObject::create(JSContext* cx, + Handle tag, + HandleObject stack, + HandleObject proto) { + Rooted obj( + cx, NewObjectWithGivenProto(cx, proto)); + if (!obj) { + return nullptr; + } + const TagType* tagType = tag->tagType(); + + // Allocate the data buffer before initializing the object so that an OOM + // does not result in a partially constructed object. + uint8_t* data = (uint8_t*)js_calloc(tagType->size_); + if (!data) { + return nullptr; + } + + MOZ_ASSERT(obj->isNewborn()); + obj->initFixedSlot(TAG_SLOT, ObjectValue(*tag)); + tagType->AddRef(); + obj->initFixedSlot(TYPE_SLOT, PrivateValue((void*)tagType)); + InitReservedSlot(obj, DATA_SLOT, data, tagType->size_, + MemoryUse::WasmExceptionData); + obj->initFixedSlot(STACK_SLOT, ObjectOrNullValue(stack)); + + MOZ_ASSERT(!obj->isNewborn()); + + return obj; +} + +bool WasmExceptionObject::isNewborn() const { + MOZ_ASSERT(is()); + return getReservedSlot(DATA_SLOT).isUndefined(); +} + +const JSPropertySpec WasmExceptionObject::properties[] = { + JS_PSG("stack", WasmExceptionObject::getStack, 0), + JS_STRING_SYM_PS(toStringTag, "WebAssembly.Exception", JSPROP_READONLY), + JS_PS_END}; + +/* static */ +bool WasmExceptionObject::isImpl(JSContext* cx, const CallArgs& args) { + Rooted exnObj( + cx, &args.thisv().toObject().as()); + + if (!args.requireAtLeast(cx, "WebAssembly.Exception.is", 1)) { + return false; + } + + if (!IsTag(args[0])) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_ARG); + return false; + } + + Rooted exnTag(cx, + &args.get(0).toObject().as()); + args.rval().setBoolean(exnTag.get() == &exnObj->tag()); + + return true; +} + +/* static */ +bool WasmExceptionObject::isMethod(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +/* static */ +bool WasmExceptionObject::getArgImpl(JSContext* cx, const CallArgs& args) { + Rooted exnObj( + cx, &args.thisv().toObject().as()); + + if (!args.requireAtLeast(cx, "WebAssembly.Exception.getArg", 2)) { + return false; + } + + if (!IsTag(args[0])) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_ARG); + return false; + } + + Rooted exnTag(cx, + &args.get(0).toObject().as()); + if (exnTag.get() != &exnObj->tag()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EXN_TAG); + return false; + } + + uint32_t index; + if (!EnforceRangeU32(cx, args.get(1), "Exception", "getArg index", &index)) { + return false; + } + + const wasm::ValTypeVector& params = exnTag->valueTypes(); + if (index >= params.length()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JSMSG_WASM_BAD_RANGE, + "Exception", "getArg index"); + return false; + } + + uint32_t offset = exnTag->tagType()->argOffsets_[index]; + RootedValue result(cx); + if (!exnObj->loadValue(cx, offset, params[index], &result)) { + return false; + } + args.rval().set(result); + return true; +} + +/* static */ +bool WasmExceptionObject::getArg(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +/* static */ +bool WasmExceptionObject::getStack_impl(JSContext* cx, const CallArgs& args) { + Rooted exnObj( + cx, &args.thisv().toObject().as()); + RootedObject savedFrameObj(cx, exnObj->stack()); + if (!savedFrameObj) { + args.rval().setUndefined(); + return true; + } + JSPrincipals* principals = exnObj->realm()->principals(); + RootedString stackString(cx); + if (!BuildStackString(cx, principals, savedFrameObj, &stackString)) { + return false; + } + args.rval().setString(stackString); + return true; +} + +/* static */ +bool WasmExceptionObject::getStack(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +JSObject* WasmExceptionObject::stack() const { + return getReservedSlot(STACK_SLOT).toObjectOrNull(); +} + +uint8_t* WasmExceptionObject::typedMem() const { + return (uint8_t*)getReservedSlot(DATA_SLOT).toPrivate(); +} + +bool WasmExceptionObject::loadValue(JSContext* cx, size_t offset, + wasm::ValType type, MutableHandleValue vp) { + if (!type.isExposable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + return ToJSValue(cx, typedMem() + offset, type, vp); +} + +bool WasmExceptionObject::initValue(JSContext* cx, size_t offset, + wasm::ValType type, HandleValue value) { + if (!type.isExposable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + + // Avoid rooting hazard of `this` being live across `fromJSValue` + // which may GC. + uint8_t* dest = typedMem() + offset; + RootedVal val(cx); + if (!Val::fromJSValue(cx, type, value, &val)) { + return false; + } + val.get().writeToHeapLocation(dest); + return true; +} + +const JSFunctionSpec WasmExceptionObject::methods[] = { + JS_FN("is", WasmExceptionObject::isMethod, 1, JSPROP_ENUMERATE), + JS_FN("getArg", WasmExceptionObject::getArg, 2, JSPROP_ENUMERATE), + JS_FS_END}; + +const JSFunctionSpec WasmExceptionObject::static_methods[] = {JS_FS_END}; + +const TagType* WasmExceptionObject::tagType() const { + return (const TagType*)getReservedSlot(TYPE_SLOT).toPrivate(); +} + +WasmTagObject& WasmExceptionObject::tag() const { + return getReservedSlot(TAG_SLOT).toObject().as(); +} + +// ============================================================================ +// WebAssembly.Function and methods +#ifdef ENABLE_WASM_TYPE_REFLECTIONS +static JSObject* CreateWasmFunctionPrototype(JSContext* cx, JSProtoKey key) { + // WasmFunction's prototype should inherit from JSFunction's prototype. + RootedObject jsProto(cx, &cx->global()->getFunctionPrototype()); + return GlobalObject::createBlankPrototypeInheriting(cx, &PlainObject::class_, + jsProto); +} + +[[nodiscard]] static bool IsWasmFunction(HandleValue v) { + if (!v.isObject()) { + return false; + } + if (!v.toObject().is()) { + return false; + } + return v.toObject().as().isWasm(); +} + +bool WasmFunctionTypeImpl(JSContext* cx, const CallArgs& args) { + RootedFunction function(cx, &args.thisv().toObject().as()); + Rooted instanceObj( + cx, ExportedFunctionToInstanceObject(function)); + uint32_t funcIndex = ExportedFunctionToFuncIndex(function); + Instance& instance = instanceObj->instance(); + const FuncExport& fe = + instance.metadata(instance.code().bestTier()).lookupFuncExport(funcIndex); + const FuncType& funcType = instance.metadata().getFuncExportType(fe); + RootedObject typeObj(cx, FuncTypeToObject(cx, funcType)); + if (!typeObj) { + return false; + } + args.rval().setObject(*typeObj); + return true; +} + +bool WasmFunctionType(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + return CallNonGenericMethod(cx, args); +} + +JSFunction* WasmFunctionCreate(JSContext* cx, HandleFunction func, + wasm::ValTypeVector&& params, + wasm::ValTypeVector&& results, + HandleObject proto) { + MOZ_RELEASE_ASSERT(!IsWasmExportedFunction(func)); + + // We want to import the function to a wasm module and then export it again so + // that it behaves exactly like a normal wasm function and can be used like + // one in wasm tables. We synthesize such a module below, instantiate it, and + // then return the exported function as the result. + FeatureOptions options; + ScriptedCaller scriptedCaller; + SharedCompileArgs compileArgs = + CompileArgs::buildAndReport(cx, std::move(scriptedCaller), options); + if (!compileArgs) { + return nullptr; + } + + ModuleEnvironment moduleEnv(compileArgs->features); + CompilerEnvironment compilerEnv(CompileMode::Once, Tier::Optimized, + DebugEnabled::False); + compilerEnv.computeParameters(); + + if (!moduleEnv.init()) { + return nullptr; + } + + FuncType funcType = FuncType(std::move(params), std::move(results)); + if (!moduleEnv.types->addType(std::move(funcType))) { + return nullptr; + } + + // Add an (import (func ...)) + FuncDesc funcDesc = FuncDesc(&(*moduleEnv.types)[0].funcType(), 0); + if (!moduleEnv.funcs.append(funcDesc)) { + return nullptr; + } + moduleEnv.numFuncImports = 1; + + // Add an (export (func 0)) + moduleEnv.declareFuncExported(0, /* eager */ true, /* canRefFunc */ true); + + // We will be looking up and using the function in the future by index so the + // name doesn't matter. + CacheableName fieldName; + if (!moduleEnv.exports.emplaceBack(std::move(fieldName), 0, + DefinitionKind::Function)) { + return nullptr; + } + + ModuleGenerator mg(*compileArgs, &moduleEnv, &compilerEnv, nullptr, nullptr, + nullptr); + if (!mg.init(nullptr)) { + return nullptr; + } + // We're not compiling any function definitions. + if (!mg.finishFuncDefs()) { + return nullptr; + } + SharedBytes shareableBytes = js_new(); + if (!shareableBytes) { + return nullptr; + } + SharedModule module = mg.finishModule(*shareableBytes); + if (!module) { + return nullptr; + } + + // Instantiate the module. + Rooted imports(cx); + if (!imports.get().funcs.append(func)) { + return nullptr; + } + Rooted instance(cx); + if (!module->instantiate(cx, imports.get(), nullptr, &instance)) { + MOZ_ASSERT(cx->isThrowingOutOfMemory()); + return nullptr; + } + + // Get the exported function which wraps the JS function to return. + RootedFunction wasmFunc(cx); + if (!instance->getExportedFunction(cx, instance, 0, &wasmFunc)) { + return nullptr; + } + return wasmFunc; +} + +bool WasmFunctionConstruct(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + if (!ThrowIfNotConstructing(cx, args, "WebAssembly.Function")) { + return false; + } + + if (!args.requireAtLeast(cx, "WebAssembly.Function", 2)) { + return false; + } + + if (!args[0].isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_DESC_ARG, "function"); + return false; + } + RootedObject typeObj(cx, &args[0].toObject()); + + // Extract properties in lexicographic order per spec. + + RootedValue parametersVal(cx); + if (!JS_GetProperty(cx, typeObj, "parameters", ¶metersVal)) { + return false; + } + + ValTypeVector params; + if (!ParseValTypes(cx, parametersVal, params)) { + return false; + } + + RootedValue resultsVal(cx); + if (!JS_GetProperty(cx, typeObj, "results", &resultsVal)) { + return false; + } + + ValTypeVector results; + if (!ParseValTypes(cx, resultsVal, results)) { + return false; + } + + // Get the target function + + if (!args[1].isObject() || !args[1].toObject().is() || + IsWasmFunction(args[1])) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_FUNCTION_VALUE); + return false; + } + RootedFunction func(cx, &args[1].toObject().as()); + + RootedObject proto( + cx, GetWasmConstructorPrototype(cx, args, JSProto_WasmFunction)); + if (!proto) { + ReportOutOfMemory(cx); + return false; + } + + RootedFunction wasmFunc(cx, WasmFunctionCreate(cx, func, std::move(params), + std::move(results), proto)); + if (!wasmFunc) { + ReportOutOfMemory(cx); + return false; + } + args.rval().setObject(*wasmFunc); + + return true; +} + +static constexpr char WasmFunctionName[] = "Function"; + +static JSObject* CreateWasmFunctionConstructor(JSContext* cx, JSProtoKey key) { + RootedObject proto(cx, &cx->global()->getFunctionConstructor()); + + Rooted className( + cx, Atomize(cx, WasmFunctionName, strlen(WasmFunctionName))); + if (!className) { + return nullptr; + } + return NewFunctionWithProto(cx, WasmFunctionConstruct, 1, + FunctionFlags::NATIVE_CTOR, nullptr, className, + proto, gc::AllocKind::FUNCTION, TenuredObject); +} + +const JSFunctionSpec WasmFunctionMethods[] = { + JS_FN("type", WasmFunctionType, 0, 0), JS_FS_END}; + +const ClassSpec WasmFunctionClassSpec = {CreateWasmFunctionConstructor, + CreateWasmFunctionPrototype, + nullptr, + nullptr, + WasmFunctionMethods, + nullptr, + nullptr, + ClassSpec::DontDefineConstructor}; + +const JSClass js::WasmFunctionClass = { + "WebAssembly.Function", 0, JS_NULL_CLASS_OPS, &WasmFunctionClassSpec}; + +#endif + +// ============================================================================ +// WebAssembly class and static methods + +static bool WebAssembly_toSource(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + args.rval().setString(cx->names().WebAssembly); + return true; +} + +static bool RejectWithPendingException(JSContext* cx, + Handle promise) { + if (!cx->isExceptionPending()) { + return false; + } + + RootedValue rejectionValue(cx); + if (!GetAndClearException(cx, &rejectionValue)) { + return false; + } + + return PromiseObject::reject(cx, promise, rejectionValue); +} + +static bool Reject(JSContext* cx, const CompileArgs& args, + Handle promise, const UniqueChars& error) { + if (!error) { + ReportOutOfMemory(cx); + return RejectWithPendingException(cx, promise); + } + + RootedObject stack(cx, promise->allocationSite()); + RootedString fileName(cx); + if (const char* filename = args.scriptedCaller.filename.get()) { + fileName = + JS_NewStringCopyUTF8N(cx, JS::UTF8Chars(filename, strlen(filename))); + } else { + fileName = JS_GetEmptyString(cx); + } + if (!fileName) { + return false; + } + + unsigned line = args.scriptedCaller.line; + + // Ideally we'd report a JSMSG_WASM_COMPILE_ERROR here, but there's no easy + // way to create an ErrorObject for an arbitrary error code with multiple + // replacements. + UniqueChars str(JS_smprintf("wasm validation error: %s", error.get())); + if (!str) { + return false; + } + + size_t len = strlen(str.get()); + RootedString message(cx, NewStringCopyN(cx, str.get(), len)); + if (!message) { + return false; + } + + // There's no error |cause| available here. + auto cause = JS::NothingHandleValue; + + RootedObject errorObj( + cx, ErrorObject::create(cx, JSEXN_WASMCOMPILEERROR, stack, fileName, 0, + line, 0, nullptr, message, cause)); + if (!errorObj) { + return false; + } + + RootedValue rejectionValue(cx, ObjectValue(*errorObj)); + return PromiseObject::reject(cx, promise, rejectionValue); +} + +static void LogAsync(JSContext* cx, const char* funcName, + const Module& module) { + Log(cx, "async %s succeeded%s", funcName, + module.loggingDeserialized() ? " (loaded from cache)" : ""); +} + +enum class Ret { Pair, Instance }; + +class AsyncInstantiateTask : public OffThreadPromiseTask { + SharedModule module_; + PersistentRooted imports_; + Ret ret_; + + public: + AsyncInstantiateTask(JSContext* cx, const Module& module, Ret ret, + Handle promise) + : OffThreadPromiseTask(cx, promise), + module_(&module), + imports_(cx), + ret_(ret) {} + + ImportValues& imports() { return imports_.get(); } + + bool resolve(JSContext* cx, Handle promise) override { + RootedObject instanceProto( + cx, &cx->global()->getPrototype(JSProto_WasmInstance)); + + Rooted instanceObj(cx); + if (!module_->instantiate(cx, imports_.get(), instanceProto, + &instanceObj)) { + return RejectWithPendingException(cx, promise); + } + + RootedValue resolutionValue(cx); + if (ret_ == Ret::Instance) { + resolutionValue = ObjectValue(*instanceObj); + } else { + RootedObject resultObj(cx, JS_NewPlainObject(cx)); + if (!resultObj) { + return RejectWithPendingException(cx, promise); + } + + RootedObject moduleProto(cx, + &cx->global()->getPrototype(JSProto_WasmModule)); + RootedObject moduleObj( + cx, WasmModuleObject::create(cx, *module_, moduleProto)); + if (!moduleObj) { + return RejectWithPendingException(cx, promise); + } + + RootedValue val(cx, ObjectValue(*moduleObj)); + if (!JS_DefineProperty(cx, resultObj, "module", val, JSPROP_ENUMERATE)) { + return RejectWithPendingException(cx, promise); + } + + val = ObjectValue(*instanceObj); + if (!JS_DefineProperty(cx, resultObj, "instance", val, + JSPROP_ENUMERATE)) { + return RejectWithPendingException(cx, promise); + } + + resolutionValue = ObjectValue(*resultObj); + } + + if (!PromiseObject::resolve(cx, promise, resolutionValue)) { + return RejectWithPendingException(cx, promise); + } + + LogAsync(cx, "instantiate", *module_); + return true; + } +}; + +static bool AsyncInstantiate(JSContext* cx, const Module& module, + HandleObject importObj, Ret ret, + Handle promise) { + auto task = js::MakeUnique(cx, module, ret, promise); + if (!task || !task->init(cx)) { + return false; + } + + if (!GetImports(cx, module, importObj, &task->imports())) { + return RejectWithPendingException(cx, promise); + } + + task.release()->dispatchResolveAndDestroy(); + return true; +} + +static bool ResolveCompile(JSContext* cx, const Module& module, + Handle promise) { + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmModule)); + RootedObject moduleObj(cx, WasmModuleObject::create(cx, module, proto)); + if (!moduleObj) { + return RejectWithPendingException(cx, promise); + } + + RootedValue resolutionValue(cx, ObjectValue(*moduleObj)); + if (!PromiseObject::resolve(cx, promise, resolutionValue)) { + return RejectWithPendingException(cx, promise); + } + + LogAsync(cx, "compile", module); + return true; +} + +struct CompileBufferTask : PromiseHelperTask { + MutableBytes bytecode; + SharedCompileArgs compileArgs; + UniqueChars error; + UniqueCharsVector warnings; + SharedModule module; + bool instantiate; + PersistentRootedObject importObj; + + CompileBufferTask(JSContext* cx, Handle promise, + HandleObject importObj) + : PromiseHelperTask(cx, promise), + instantiate(true), + importObj(cx, importObj) {} + + CompileBufferTask(JSContext* cx, Handle promise) + : PromiseHelperTask(cx, promise), instantiate(false) {} + + bool init(JSContext* cx, const char* introducer) { + compileArgs = InitCompileArgs(cx, introducer); + if (!compileArgs) { + return false; + } + return PromiseHelperTask::init(cx); + } + + void execute() override { + module = CompileBuffer(*compileArgs, *bytecode, &error, &warnings, nullptr); + } + + bool resolve(JSContext* cx, Handle promise) override { + if (!ReportCompileWarnings(cx, warnings)) { + return false; + } + if (!module) { + return Reject(cx, *compileArgs, promise, error); + } + if (instantiate) { + return AsyncInstantiate(cx, *module, importObj, Ret::Pair, promise); + } + return ResolveCompile(cx, *module, promise); + } +}; + +static bool RejectWithPendingException(JSContext* cx, + Handle promise, + CallArgs& callArgs) { + if (!RejectWithPendingException(cx, promise)) { + return false; + } + + callArgs.rval().setObject(*promise); + return true; +} + +static bool EnsurePromiseSupport(JSContext* cx) { + if (!cx->runtime()->offThreadPromiseState.ref().initialized()) { + JS_ReportErrorASCII( + cx, "WebAssembly Promise APIs not supported in this runtime."); + return false; + } + return true; +} + +static bool GetBufferSource(JSContext* cx, CallArgs callArgs, const char* name, + MutableBytes* bytecode) { + if (!callArgs.requireAtLeast(cx, name, 1)) { + return false; + } + + if (!callArgs[0].isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_BUF_ARG); + return false; + } + + return GetBufferSource(cx, &callArgs[0].toObject(), JSMSG_WASM_BAD_BUF_ARG, + bytecode); +} + +static bool WebAssembly_compile(JSContext* cx, unsigned argc, Value* vp) { + if (!EnsurePromiseSupport(cx)) { + return false; + } + + Log(cx, "async compile() started"); + + Rooted promise(cx, PromiseObject::createSkippingExecutor(cx)); + if (!promise) { + return false; + } + + CallArgs callArgs = CallArgsFromVp(argc, vp); + + if (!cx->isRuntimeCodeGenEnabled(JS::RuntimeCode::WASM, nullptr)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_CSP_BLOCKED_WASM, "WebAssembly.compile"); + return RejectWithPendingException(cx, promise, callArgs); + } + + auto task = cx->make_unique(cx, promise); + if (!task || !task->init(cx, "WebAssembly.compile")) { + return false; + } + + if (!GetBufferSource(cx, callArgs, "WebAssembly.compile", &task->bytecode)) { + return RejectWithPendingException(cx, promise, callArgs); + } + + if (!StartOffThreadPromiseHelperTask(cx, std::move(task))) { + return false; + } + + callArgs.rval().setObject(*promise); + return true; +} + +static bool GetInstantiateArgs(JSContext* cx, CallArgs callArgs, + MutableHandleObject firstArg, + MutableHandleObject importObj) { + if (!callArgs.requireAtLeast(cx, "WebAssembly.instantiate", 1)) { + return false; + } + + if (!callArgs[0].isObject()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_BUF_MOD_ARG); + return false; + } + + firstArg.set(&callArgs[0].toObject()); + + return GetImportArg(cx, callArgs, importObj); +} + +static bool WebAssembly_instantiate(JSContext* cx, unsigned argc, Value* vp) { + if (!EnsurePromiseSupport(cx)) { + return false; + } + + Log(cx, "async instantiate() started"); + + Rooted promise(cx, PromiseObject::createSkippingExecutor(cx)); + if (!promise) { + return false; + } + + CallArgs callArgs = CallArgsFromVp(argc, vp); + + RootedObject firstArg(cx); + RootedObject importObj(cx); + if (!GetInstantiateArgs(cx, callArgs, &firstArg, &importObj)) { + return RejectWithPendingException(cx, promise, callArgs); + } + + const Module* module; + if (IsModuleObject(firstArg, &module)) { + if (!AsyncInstantiate(cx, *module, importObj, Ret::Instance, promise)) { + return false; + } + } else { + if (!cx->isRuntimeCodeGenEnabled(JS::RuntimeCode::WASM, nullptr)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_CSP_BLOCKED_WASM, + "WebAssembly.instantiate"); + return RejectWithPendingException(cx, promise, callArgs); + } + + auto task = cx->make_unique(cx, promise, importObj); + if (!task || !task->init(cx, "WebAssembly.instantiate")) { + return false; + } + + if (!GetBufferSource(cx, firstArg, JSMSG_WASM_BAD_BUF_MOD_ARG, + &task->bytecode)) { + return RejectWithPendingException(cx, promise, callArgs); + } + + if (!StartOffThreadPromiseHelperTask(cx, std::move(task))) { + return false; + } + } + + callArgs.rval().setObject(*promise); + return true; +} + +static bool WebAssembly_validate(JSContext* cx, unsigned argc, Value* vp) { + CallArgs callArgs = CallArgsFromVp(argc, vp); + + MutableBytes bytecode; + if (!GetBufferSource(cx, callArgs, "WebAssembly.validate", &bytecode)) { + return false; + } + + FeatureOptions options; + UniqueChars error; + bool validated = Validate(cx, *bytecode, options, &error); + + // If the reason for validation failure was OOM (signalled by null error + // message), report out-of-memory so that validate's return is always + // correct. + if (!validated && !error) { + ReportOutOfMemory(cx); + return false; + } + + if (error) { + MOZ_ASSERT(!validated); + Log(cx, "validate() failed with: %s", error.get()); + } + + callArgs.rval().setBoolean(validated); + return true; +} + +static bool EnsureStreamSupport(JSContext* cx) { + // This should match wasm::StreamingCompilationAvailable(). + + if (!EnsurePromiseSupport(cx)) { + return false; + } + + if (!CanUseExtraThreads()) { + JS_ReportErrorASCII( + cx, "WebAssembly.compileStreaming not supported with --no-threads"); + return false; + } + + if (!cx->runtime()->consumeStreamCallback) { + JS_ReportErrorASCII(cx, + "WebAssembly streaming not supported in this runtime"); + return false; + } + + return true; +} + +// This value is chosen and asserted to be disjoint from any host error code. +static const size_t StreamOOMCode = 0; + +static bool RejectWithStreamErrorNumber(JSContext* cx, size_t errorCode, + Handle promise) { + if (errorCode == StreamOOMCode) { + ReportOutOfMemory(cx); + return false; + } + + cx->runtime()->reportStreamErrorCallback(cx, errorCode); + return RejectWithPendingException(cx, promise); +} + +class CompileStreamTask : public PromiseHelperTask, public JS::StreamConsumer { + // The stream progresses monotonically through these states; the helper + // thread wait()s for streamState_ to reach Closed. + enum StreamState { Env, Code, Tail, Closed }; + ExclusiveWaitableData streamState_; + + // Immutable: + const bool instantiate_; + const PersistentRootedObject importObj_; + + // Immutable after noteResponseURLs() which is called at most once before + // first call on stream thread: + const MutableCompileArgs compileArgs_; + + // Immutable after Env state: + Bytes envBytes_; + SectionRange codeSection_; + + // The code section vector is resized once during the Env state and filled + // in chunk by chunk during the Code state, updating the end-pointer after + // each chunk: + Bytes codeBytes_; + uint8_t* codeBytesEnd_; + ExclusiveBytesPtr exclusiveCodeBytesEnd_; + + // Immutable after Tail state: + Bytes tailBytes_; + ExclusiveStreamEndData exclusiveStreamEnd_; + + // Written once before Closed state and read in Closed state on main thread: + SharedModule module_; + Maybe streamError_; + UniqueChars compileError_; + UniqueCharsVector warnings_; + + // Set on stream thread and read racily on helper thread to abort compilation: + Atomic streamFailed_; + + // Called on some thread before consumeChunk(), streamEnd(), streamError()): + + void noteResponseURLs(const char* url, const char* sourceMapUrl) override { + if (url) { + compileArgs_->scriptedCaller.filename = DuplicateString(url); + compileArgs_->scriptedCaller.filenameIsURL = true; + } + if (sourceMapUrl) { + compileArgs_->sourceMapURL = DuplicateString(sourceMapUrl); + } + } + + // Called on a stream thread: + + // Until StartOffThreadPromiseHelperTask succeeds, we are responsible for + // dispatching ourselves back to the JS thread. + // + // Warning: After this function returns, 'this' can be deleted at any time, so + // the caller must immediately return from the stream callback. + void setClosedAndDestroyBeforeHelperThreadStarted() { + streamState_.lock().get() = Closed; + dispatchResolveAndDestroy(); + } + + // See setClosedAndDestroyBeforeHelperThreadStarted() comment. + bool rejectAndDestroyBeforeHelperThreadStarted(size_t errorNumber) { + MOZ_ASSERT(streamState_.lock() == Env); + MOZ_ASSERT(!streamError_); + streamError_ = Some(errorNumber); + setClosedAndDestroyBeforeHelperThreadStarted(); + return false; + } + + // Once StartOffThreadPromiseHelperTask succeeds, the helper thread will + // dispatchResolveAndDestroy() after execute() returns, but execute() + // wait()s for state to be Closed. + // + // Warning: After this function returns, 'this' can be deleted at any time, so + // the caller must immediately return from the stream callback. + void setClosedAndDestroyAfterHelperThreadStarted() { + auto streamState = streamState_.lock(); + MOZ_ASSERT(streamState != Closed); + streamState.get() = Closed; + streamState.notify_one(/* stream closed */); + } + + // See setClosedAndDestroyAfterHelperThreadStarted() comment. + bool rejectAndDestroyAfterHelperThreadStarted(size_t errorNumber) { + MOZ_ASSERT(!streamError_); + streamError_ = Some(errorNumber); + streamFailed_ = true; + exclusiveCodeBytesEnd_.lock().notify_one(); + exclusiveStreamEnd_.lock().notify_one(); + setClosedAndDestroyAfterHelperThreadStarted(); + return false; + } + + bool consumeChunk(const uint8_t* begin, size_t length) override { + switch (streamState_.lock().get()) { + case Env: { + if (!envBytes_.append(begin, length)) { + return rejectAndDestroyBeforeHelperThreadStarted(StreamOOMCode); + } + + if (!StartsCodeSection(envBytes_.begin(), envBytes_.end(), + &codeSection_)) { + return true; + } + + uint32_t extraBytes = envBytes_.length() - codeSection_.start; + if (extraBytes) { + envBytes_.shrinkTo(codeSection_.start); + } + + if (codeSection_.size > MaxCodeSectionBytes) { + return rejectAndDestroyBeforeHelperThreadStarted(StreamOOMCode); + } + + if (!codeBytes_.resize(codeSection_.size)) { + return rejectAndDestroyBeforeHelperThreadStarted(StreamOOMCode); + } + + codeBytesEnd_ = codeBytes_.begin(); + exclusiveCodeBytesEnd_.lock().get() = codeBytesEnd_; + + if (!StartOffThreadPromiseHelperTask(this)) { + return rejectAndDestroyBeforeHelperThreadStarted(StreamOOMCode); + } + + // Set the state to Code iff StartOffThreadPromiseHelperTask() + // succeeds so that the state tells us whether we are before or + // after the helper thread started. + streamState_.lock().get() = Code; + + if (extraBytes) { + return consumeChunk(begin + length - extraBytes, extraBytes); + } + + return true; + } + case Code: { + size_t copyLength = + std::min(length, codeBytes_.end() - codeBytesEnd_); + memcpy(codeBytesEnd_, begin, copyLength); + codeBytesEnd_ += copyLength; + + { + auto codeStreamEnd = exclusiveCodeBytesEnd_.lock(); + codeStreamEnd.get() = codeBytesEnd_; + codeStreamEnd.notify_one(); + } + + if (codeBytesEnd_ != codeBytes_.end()) { + return true; + } + + streamState_.lock().get() = Tail; + + if (uint32_t extraBytes = length - copyLength) { + return consumeChunk(begin + copyLength, extraBytes); + } + + return true; + } + case Tail: { + if (!tailBytes_.append(begin, length)) { + return rejectAndDestroyAfterHelperThreadStarted(StreamOOMCode); + } + + return true; + } + case Closed: + MOZ_CRASH("consumeChunk() in Closed state"); + } + MOZ_CRASH("unreachable"); + } + + void streamEnd(JS::OptimizedEncodingListener* tier2Listener) override { + switch (streamState_.lock().get()) { + case Env: { + SharedBytes bytecode = js_new(std::move(envBytes_)); + if (!bytecode) { + rejectAndDestroyBeforeHelperThreadStarted(StreamOOMCode); + return; + } + module_ = CompileBuffer(*compileArgs_, *bytecode, &compileError_, + &warnings_, nullptr); + setClosedAndDestroyBeforeHelperThreadStarted(); + return; + } + case Code: + case Tail: + // Unlock exclusiveStreamEnd_ before locking streamState_. + { + auto streamEnd = exclusiveStreamEnd_.lock(); + MOZ_ASSERT(!streamEnd->reached); + streamEnd->reached = true; + streamEnd->tailBytes = &tailBytes_; + streamEnd->tier2Listener = tier2Listener; + streamEnd.notify_one(); + } + setClosedAndDestroyAfterHelperThreadStarted(); + return; + case Closed: + MOZ_CRASH("streamEnd() in Closed state"); + } + } + + void streamError(size_t errorCode) override { + MOZ_ASSERT(errorCode != StreamOOMCode); + switch (streamState_.lock().get()) { + case Env: + rejectAndDestroyBeforeHelperThreadStarted(errorCode); + return; + case Tail: + case Code: + rejectAndDestroyAfterHelperThreadStarted(errorCode); + return; + case Closed: + MOZ_CRASH("streamError() in Closed state"); + } + } + + void consumeOptimizedEncoding(const uint8_t* begin, size_t length) override { + module_ = Module::deserialize(begin, length); + + MOZ_ASSERT(streamState_.lock().get() == Env); + setClosedAndDestroyBeforeHelperThreadStarted(); + } + + // Called on a helper thread: + + void execute() override { + module_ = CompileStreaming(*compileArgs_, envBytes_, codeBytes_, + exclusiveCodeBytesEnd_, exclusiveStreamEnd_, + streamFailed_, &compileError_, &warnings_); + + // When execute() returns, the CompileStreamTask will be dispatched + // back to its JS thread to call resolve() and then be destroyed. We + // can't let this happen until the stream has been closed lest + // consumeChunk() or streamEnd() be called on a dead object. + auto streamState = streamState_.lock(); + while (streamState != Closed) { + streamState.wait(/* stream closed */); + } + } + + // Called on a JS thread after streaming compilation completes/errors: + + bool resolve(JSContext* cx, Handle promise) override { + MOZ_ASSERT(streamState_.lock() == Closed); + + if (!ReportCompileWarnings(cx, warnings_)) { + return false; + } + if (module_) { + MOZ_ASSERT(!streamFailed_ && !streamError_ && !compileError_); + if (instantiate_) { + return AsyncInstantiate(cx, *module_, importObj_, Ret::Pair, promise); + } + return ResolveCompile(cx, *module_, promise); + } + + if (streamError_) { + return RejectWithStreamErrorNumber(cx, *streamError_, promise); + } + + return Reject(cx, *compileArgs_, promise, compileError_); + } + + public: + CompileStreamTask(JSContext* cx, Handle promise, + CompileArgs& compileArgs, bool instantiate, + HandleObject importObj) + : PromiseHelperTask(cx, promise), + streamState_(mutexid::WasmStreamStatus, Env), + instantiate_(instantiate), + importObj_(cx, importObj), + compileArgs_(&compileArgs), + codeSection_{}, + codeBytesEnd_(nullptr), + exclusiveCodeBytesEnd_(mutexid::WasmCodeBytesEnd, nullptr), + exclusiveStreamEnd_(mutexid::WasmStreamEnd), + streamFailed_(false) { + MOZ_ASSERT_IF(importObj_, instantiate_); + } +}; + +// A short-lived object that captures the arguments of a +// WebAssembly.{compileStreaming,instantiateStreaming} while waiting for +// the Promise to resolve to a (hopefully) Promise. +class ResolveResponseClosure : public NativeObject { + static const unsigned COMPILE_ARGS_SLOT = 0; + static const unsigned PROMISE_OBJ_SLOT = 1; + static const unsigned INSTANTIATE_SLOT = 2; + static const unsigned IMPORT_OBJ_SLOT = 3; + static const JSClassOps classOps_; + + static void finalize(JS::GCContext* gcx, JSObject* obj) { + auto& closure = obj->as(); + gcx->release(obj, &closure.compileArgs(), + MemoryUse::WasmResolveResponseClosure); + } + + public: + static const unsigned RESERVED_SLOTS = 4; + static const JSClass class_; + + static ResolveResponseClosure* create(JSContext* cx, const CompileArgs& args, + HandleObject promise, bool instantiate, + HandleObject importObj) { + MOZ_ASSERT_IF(importObj, instantiate); + + AutoSetNewObjectMetadata metadata(cx); + auto* obj = NewObjectWithGivenProto(cx, nullptr); + if (!obj) { + return nullptr; + } + + args.AddRef(); + InitReservedSlot(obj, COMPILE_ARGS_SLOT, const_cast(&args), + MemoryUse::WasmResolveResponseClosure); + obj->setReservedSlot(PROMISE_OBJ_SLOT, ObjectValue(*promise)); + obj->setReservedSlot(INSTANTIATE_SLOT, BooleanValue(instantiate)); + obj->setReservedSlot(IMPORT_OBJ_SLOT, ObjectOrNullValue(importObj)); + return obj; + } + + CompileArgs& compileArgs() const { + return *(CompileArgs*)getReservedSlot(COMPILE_ARGS_SLOT).toPrivate(); + } + PromiseObject& promise() const { + return getReservedSlot(PROMISE_OBJ_SLOT).toObject().as(); + } + bool instantiate() const { + return getReservedSlot(INSTANTIATE_SLOT).toBoolean(); + } + JSObject* importObj() const { + return getReservedSlot(IMPORT_OBJ_SLOT).toObjectOrNull(); + } +}; + +const JSClassOps ResolveResponseClosure::classOps_ = { + nullptr, // addProperty + nullptr, // delProperty + nullptr, // enumerate + nullptr, // newEnumerate + nullptr, // resolve + nullptr, // mayResolve + ResolveResponseClosure::finalize, // finalize + nullptr, // call + nullptr, // construct + nullptr, // trace +}; + +const JSClass ResolveResponseClosure::class_ = { + "WebAssembly ResolveResponseClosure", + JSCLASS_DELAY_METADATA_BUILDER | + JSCLASS_HAS_RESERVED_SLOTS(ResolveResponseClosure::RESERVED_SLOTS) | + JSCLASS_FOREGROUND_FINALIZE, + &ResolveResponseClosure::classOps_, +}; + +static ResolveResponseClosure* ToResolveResponseClosure(CallArgs args) { + return &args.callee() + .as() + .getExtendedSlot(0) + .toObject() + .as(); +} + +static bool RejectWithErrorNumber(JSContext* cx, uint32_t errorNumber, + Handle promise) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, errorNumber); + return RejectWithPendingException(cx, promise); +} + +static bool ResolveResponse_OnFulfilled(JSContext* cx, unsigned argc, + Value* vp) { + CallArgs callArgs = CallArgsFromVp(argc, vp); + + Rooted closure(cx, + ToResolveResponseClosure(callArgs)); + Rooted promise(cx, &closure->promise()); + CompileArgs& compileArgs = closure->compileArgs(); + bool instantiate = closure->instantiate(); + Rooted importObj(cx, closure->importObj()); + + auto task = cx->make_unique(cx, promise, compileArgs, + instantiate, importObj); + if (!task || !task->init(cx)) { + return false; + } + + if (!callArgs.get(0).isObject()) { + return RejectWithErrorNumber(cx, JSMSG_WASM_BAD_RESPONSE_VALUE, promise); + } + + RootedObject response(cx, &callArgs.get(0).toObject()); + if (!cx->runtime()->consumeStreamCallback(cx, response, JS::MimeType::Wasm, + task.get())) { + return RejectWithPendingException(cx, promise); + } + + (void)task.release(); + + callArgs.rval().setUndefined(); + return true; +} + +static bool ResolveResponse_OnRejected(JSContext* cx, unsigned argc, + Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + Rooted closure(cx, ToResolveResponseClosure(args)); + Rooted promise(cx, &closure->promise()); + + if (!PromiseObject::reject(cx, promise, args.get(0))) { + return false; + } + + args.rval().setUndefined(); + return true; +} + +static bool ResolveResponse(JSContext* cx, CallArgs callArgs, + Handle promise, + bool instantiate = false, + HandleObject importObj = nullptr) { + MOZ_ASSERT_IF(importObj, instantiate); + + const char* introducer = instantiate ? "WebAssembly.instantiateStreaming" + : "WebAssembly.compileStreaming"; + + SharedCompileArgs compileArgs = InitCompileArgs(cx, introducer); + if (!compileArgs) { + return false; + } + + RootedObject closure( + cx, ResolveResponseClosure::create(cx, *compileArgs, promise, instantiate, + importObj)); + if (!closure) { + return false; + } + + RootedFunction onResolved( + cx, NewNativeFunction(cx, ResolveResponse_OnFulfilled, 1, nullptr, + gc::AllocKind::FUNCTION_EXTENDED, GenericObject)); + if (!onResolved) { + return false; + } + + RootedFunction onRejected( + cx, NewNativeFunction(cx, ResolveResponse_OnRejected, 1, nullptr, + gc::AllocKind::FUNCTION_EXTENDED, GenericObject)); + if (!onRejected) { + return false; + } + + onResolved->setExtendedSlot(0, ObjectValue(*closure)); + onRejected->setExtendedSlot(0, ObjectValue(*closure)); + + RootedObject resolve(cx, + PromiseObject::unforgeableResolve(cx, callArgs.get(0))); + if (!resolve) { + return false; + } + + return JS::AddPromiseReactions(cx, resolve, onResolved, onRejected); +} + +static bool WebAssembly_compileStreaming(JSContext* cx, unsigned argc, + Value* vp) { + if (!EnsureStreamSupport(cx)) { + return false; + } + + Log(cx, "async compileStreaming() started"); + + Rooted promise(cx, PromiseObject::createSkippingExecutor(cx)); + if (!promise) { + return false; + } + + CallArgs callArgs = CallArgsFromVp(argc, vp); + + if (!cx->isRuntimeCodeGenEnabled(JS::RuntimeCode::WASM, nullptr)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_CSP_BLOCKED_WASM, + "WebAssembly.compileStreaming"); + return RejectWithPendingException(cx, promise, callArgs); + } + + if (!ResolveResponse(cx, callArgs, promise)) { + return RejectWithPendingException(cx, promise, callArgs); + } + + callArgs.rval().setObject(*promise); + return true; +} + +static bool WebAssembly_instantiateStreaming(JSContext* cx, unsigned argc, + Value* vp) { + if (!EnsureStreamSupport(cx)) { + return false; + } + + Log(cx, "async instantiateStreaming() started"); + + Rooted promise(cx, PromiseObject::createSkippingExecutor(cx)); + if (!promise) { + return false; + } + + CallArgs callArgs = CallArgsFromVp(argc, vp); + + if (!cx->isRuntimeCodeGenEnabled(JS::RuntimeCode::WASM, nullptr)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_CSP_BLOCKED_WASM, + "WebAssembly.instantiateStreaming"); + return RejectWithPendingException(cx, promise, callArgs); + } + + RootedObject firstArg(cx); + RootedObject importObj(cx); + if (!GetInstantiateArgs(cx, callArgs, &firstArg, &importObj)) { + return RejectWithPendingException(cx, promise, callArgs); + } + + if (!ResolveResponse(cx, callArgs, promise, true, importObj)) { + return RejectWithPendingException(cx, promise, callArgs); + } + + callArgs.rval().setObject(*promise); + return true; +} + +#ifdef ENABLE_WASM_MOZ_INTGEMM + +static bool WebAssembly_mozIntGemm(JSContext* cx, unsigned argc, Value* vp) { + CallArgs args = CallArgsFromVp(argc, vp); + + Rooted module(cx); + wasm::IntrinsicId ids[] = { + wasm::IntrinsicId::I8PrepareB, + wasm::IntrinsicId::I8PrepareBFromTransposed, + wasm::IntrinsicId::I8PrepareBFromQuantizedTransposed, + wasm::IntrinsicId::I8PrepareA, + wasm::IntrinsicId::I8PrepareBias, + wasm::IntrinsicId::I8MultiplyAndAddBias, + wasm::IntrinsicId::I8SelectColumnsOfB}; + if (!wasm::CompileIntrinsicModule(cx, ids, Shareable::False, &module)) { + ReportOutOfMemory(cx); + return false; + } + args.rval().set(ObjectValue(*module.get())); + return true; +} + +static const JSFunctionSpec WebAssembly_mozIntGemm_methods[] = { + JS_FN("mozIntGemm", WebAssembly_mozIntGemm, 0, JSPROP_ENUMERATE), + JS_FS_END}; + +#endif // ENABLE_WASM_MOZ_INTGEMM + +static const JSFunctionSpec WebAssembly_static_methods[] = { + JS_FN(js_toSource_str, WebAssembly_toSource, 0, 0), + JS_FN("compile", WebAssembly_compile, 1, JSPROP_ENUMERATE), + JS_FN("instantiate", WebAssembly_instantiate, 1, JSPROP_ENUMERATE), + JS_FN("validate", WebAssembly_validate, 1, JSPROP_ENUMERATE), + JS_FN("compileStreaming", WebAssembly_compileStreaming, 1, + JSPROP_ENUMERATE), + JS_FN("instantiateStreaming", WebAssembly_instantiateStreaming, 1, + JSPROP_ENUMERATE), + JS_FS_END}; + +static const JSPropertySpec WebAssembly_static_properties[] = { + JS_STRING_SYM_PS(toStringTag, "WebAssembly", JSPROP_READONLY), JS_PS_END}; + +static JSObject* CreateWebAssemblyObject(JSContext* cx, JSProtoKey key) { + MOZ_RELEASE_ASSERT(HasSupport(cx)); + + RootedObject proto(cx, &cx->global()->getObjectPrototype()); + return NewTenuredObjectWithGivenProto(cx, &WasmNamespaceObject::class_, + proto); +} + +struct NameAndProtoKey { + const char* const name; + JSProtoKey key; +}; + +static bool WebAssemblyDefineConstructor(JSContext* cx, + Handle wasm, + NameAndProtoKey entry, + MutableHandleValue ctorValue, + MutableHandleId id) { + JSObject* ctor = GlobalObject::getOrCreateConstructor(cx, entry.key); + if (!ctor) { + return false; + } + ctorValue.setObject(*ctor); + + JSAtom* className = Atomize(cx, entry.name, strlen(entry.name)); + if (!className) { + return false; + } + id.set(AtomToId(className)); + + return DefineDataProperty(cx, wasm, id, ctorValue, 0); +} + +static bool WebAssemblyClassFinish(JSContext* cx, HandleObject object, + HandleObject proto) { + Handle wasm = object.as(); + + constexpr NameAndProtoKey entries[] = { + {"Module", JSProto_WasmModule}, + {"Instance", JSProto_WasmInstance}, + {"Memory", JSProto_WasmMemory}, + {"Table", JSProto_WasmTable}, + {"Global", JSProto_WasmGlobal}, + {"CompileError", GetExceptionProtoKey(JSEXN_WASMCOMPILEERROR)}, + {"LinkError", GetExceptionProtoKey(JSEXN_WASMLINKERROR)}, + {"RuntimeError", GetExceptionProtoKey(JSEXN_WASMRUNTIMEERROR)}, +#ifdef ENABLE_WASM_TYPE_REFLECTIONS + {"Function", JSProto_WasmFunction}, +#endif + }; + RootedValue ctorValue(cx); + RootedId id(cx); + for (const auto& entry : entries) { + if (!WebAssemblyDefineConstructor(cx, wasm, entry, &ctorValue, &id)) { + return false; + } + } + + if (ExceptionsAvailable(cx)) { + constexpr NameAndProtoKey exceptionEntries[] = { + {"Tag", JSProto_WasmTag}, + {"Exception", JSProto_WasmException}, + }; + for (const auto& entry : exceptionEntries) { + if (!WebAssemblyDefineConstructor(cx, wasm, entry, &ctorValue, &id)) { + return false; + } + } + } + +#ifdef ENABLE_WASM_MOZ_INTGEMM + if (MozIntGemmAvailable(cx) && + !JS_DefineFunctions(cx, wasm, WebAssembly_mozIntGemm_methods)) { + return false; + } +#endif + + return true; +} + +static const ClassSpec WebAssemblyClassSpec = { + CreateWebAssemblyObject, nullptr, WebAssembly_static_methods, + WebAssembly_static_properties, nullptr, nullptr, + WebAssemblyClassFinish}; + +const JSClass js::WasmNamespaceObject::class_ = { + js_WebAssembly_str, JSCLASS_HAS_CACHED_PROTO(JSProto_WebAssembly), + JS_NULL_CLASS_OPS, &WebAssemblyClassSpec}; + +// Sundry diff --git a/js/src/wasm/WasmJS.h b/js/src/wasm/WasmJS.h new file mode 100644 index 0000000000..53de0f2962 --- /dev/null +++ b/js/src/wasm/WasmJS.h @@ -0,0 +1,580 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_js_h +#define wasm_js_h + +#include "mozilla/HashTable.h" // DefaultHasher +#include "mozilla/Maybe.h" // mozilla::Maybe + +#include // int32_t, int64_t, uint32_t + +#include "gc/Barrier.h" // HeapPtr +#include "gc/ZoneAllocator.h" // ZoneAllocPolicy +#include "js/AllocPolicy.h" // SystemAllocPolicy +#include "js/Class.h" // JSClassOps, ClassSpec +#include "js/GCHashTable.h" // GCHashMap, GCHashSet +#include "js/GCVector.h" // GCVector +#include "js/PropertySpec.h" // JSPropertySpec, JSFunctionSpec +#include "js/RootingAPI.h" // StableCellHasher +#include "js/SweepingAPI.h" // JS::WeakCache +#include "js/TypeDecls.h" // HandleValue, HandleObject, MutableHandleObject, MutableHandleFunction +#include "js/Vector.h" // JS::Vector +#include "js/WasmFeatures.h" +#include "vm/JSFunction.h" // JSFunction +#include "vm/NativeObject.h" // NativeObject +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmException.h" +#include "wasm/WasmExprType.h" +#include "wasm/WasmMemory.h" +#include "wasm/WasmModuleTypes.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmValType.h" +#include "wasm/WasmValue.h" + +class JSObject; +class JSTracer; +struct JSContext; + +namespace JS { +class CallArgs; +class Value; +} // namespace JS + +namespace js { + +class ArrayBufferObject; +class ArrayBufferObjectMaybeShared; +class JSStringBuilder; +class TypedArrayObject; +class WasmFunctionScope; +class WasmInstanceScope; +class WasmSharedArrayRawBuffer; + +namespace wasm { + +struct ImportValues; + +// Return whether WebAssembly can in principle be compiled on this platform (ie +// combination of hardware and OS), assuming at least one of the compilers that +// supports the platform is not disabled by other settings. +// +// This predicate must be checked and must be true to call any of the top-level +// wasm eval/compile methods. + +bool HasPlatformSupport(JSContext* cx); + +// Return whether WebAssembly is supported on this platform. This determines +// whether the WebAssembly object is exposed to JS in this context / realm and +// +// It does *not* guarantee that a compiler is actually available; that has to be +// checked separately, as it is sometimes run-time variant, depending on whether +// a debugger has been created or not. + +bool HasSupport(JSContext* cx); + +// Predicates for compiler availability. +// +// These three predicates together select zero or one baseline compiler and zero +// or one optimizing compiler, based on: what's compiled into the executable, +// what's supported on the current platform, what's selected by options, and the +// current run-time environment. As it is possible for the computed values to +// change (when a value changes in about:config or the debugger pane is shown or +// hidden), it is inadvisable to cache these values in such a way that they +// could become invalid. Generally it is cheap always to recompute them. + +bool BaselineAvailable(JSContext* cx); +bool IonAvailable(JSContext* cx); + +// Test all three. + +bool AnyCompilerAvailable(JSContext* cx); + +// Asm.JS is translated to wasm and then compiled using the wasm optimizing +// compiler; test whether this compiler is available. + +bool WasmCompilerForAsmJSAvailable(JSContext* cx); + +// Predicates for white-box compiler disablement testing. +// +// These predicates determine whether the optimizing compilers were disabled by +// features that are enabled at compile-time or run-time. They do not consider +// the hardware platform on whether other compilers are enabled. +// +// If `reason` is not null then it is populated with a string that describes +// the specific features that disable the compiler. +// +// Returns false on OOM (which happens only when a reason is requested), +// otherwise true, with the result in `*isDisabled` and optionally the reason in +// `*reason`. + +bool BaselineDisabledByFeatures(JSContext* cx, bool* isDisabled, + JSStringBuilder* reason = nullptr); +bool IonDisabledByFeatures(JSContext* cx, bool* isDisabled, + JSStringBuilder* reason = nullptr); + +// Predicates for feature availability. +// +// The following predicates check whether particular wasm features are enabled, +// and for each, whether at least one compiler is (currently) available that +// supports the feature. + +// Streaming compilation. +bool StreamingCompilationAvailable(JSContext* cx); + +// Caching of optimized code. Implies both streaming compilation and an +// optimizing compiler tier. +bool CodeCachingAvailable(JSContext* cx); + +// Shared memory and atomics. +bool ThreadsAvailable(JSContext* cx); + +#define WASM_FEATURE(NAME, ...) bool NAME##Available(JSContext* cx); +JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE) +#undef WASM_FEATURE + +// SIMD operations. +bool SimdAvailable(JSContext* cx); + +// Privileged content that can access experimental intrinsics +bool IsSimdPrivilegedContext(JSContext* cx); + +#if defined(ENABLE_WASM_SIMD) && defined(DEBUG) +// Report the result of a Simd simplification to the testing infrastructure. +void ReportSimdAnalysis(const char* data); +#endif + +// Returns true if WebAssembly as configured by compile-time flags and run-time +// options can support try/catch, throw, rethrow, and branch_on_exn (evolving). +bool ExceptionsAvailable(JSContext* cx); + +// Compiles the given binary wasm module given the ArrayBufferObject +// and links the module's imports with the given import object. + +[[nodiscard]] bool Eval(JSContext* cx, Handle code, + HandleObject importObj, + MutableHandle instanceObj); + +// Extracts the various imports from the given import object into the given +// ImportValues structure while checking the imports against the given module. +// The resulting structure can be passed to WasmModule::instantiate. + +struct ImportValues; +[[nodiscard]] bool GetImports(JSContext* cx, const Module& module, + HandleObject importObj, ImportValues* imports); + +// For testing cross-process (de)serialization, this pair of functions are +// responsible for, in the child process, compiling the given wasm bytecode +// to a wasm::Module that is serialized into the given byte array, and, in +// the parent process, deserializing the given byte array into a +// WebAssembly.Module object. + +[[nodiscard]] bool CompileAndSerialize(JSContext* cx, + const ShareableBytes& bytecode, + Bytes* serialized); + +[[nodiscard]] bool DeserializeModule(JSContext* cx, const Bytes& serialized, + MutableHandleObject module); + +// A WebAssembly "Exported Function" is the spec name for the JS function +// objects created to wrap wasm functions. This predicate returns false +// for asm.js functions which are semantically just normal JS functions +// (even if they are implemented via wasm under the hood). The accessor +// functions for extracting the instance and func-index of a wasm function +// can be used for both wasm and asm.js, however. + +bool IsWasmExportedFunction(JSFunction* fun); + +Instance& ExportedFunctionToInstance(JSFunction* fun); +WasmInstanceObject* ExportedFunctionToInstanceObject(JSFunction* fun); +uint32_t ExportedFunctionToFuncIndex(JSFunction* fun); + +bool IsSharedWasmMemoryObject(JSObject* obj); + +} // namespace wasm + +// The class of WebAssembly.Module. Each WasmModuleObject owns a +// wasm::Module. These objects are used both as content-facing JS objects and as +// internal implementation details of asm.js. + +class WasmModuleObject : public NativeObject { + static const unsigned MODULE_SLOT = 0; + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + static void finalize(JS::GCContext* gcx, JSObject* obj); + static bool imports(JSContext* cx, unsigned argc, Value* vp); + static bool exports(JSContext* cx, unsigned argc, Value* vp); + static bool customSections(JSContext* cx, unsigned argc, Value* vp); + + public: + static const unsigned RESERVED_SLOTS = 1; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + static WasmModuleObject* create(JSContext* cx, const wasm::Module& module, + HandleObject proto); + const wasm::Module& module() const; +}; + +// The class of WebAssembly.Global. This wraps a storage location, and there is +// a per-agent one-to-one relationship between the WasmGlobalObject and the +// storage location (the Cell) it wraps: if a module re-exports an imported +// global, the imported and exported WasmGlobalObjects are the same, and if a +// module exports a global twice, the two exported WasmGlobalObjects are the +// same. + +// TODO/AnyRef-boxing: With boxed immediates and strings, JSObject* is no longer +// the most appropriate representation for Cell::anyref. +STATIC_ASSERT_ANYREF_IS_JSOBJECT; + +class WasmGlobalObject : public NativeObject { + static const unsigned MUTABLE_SLOT = 0; + static const unsigned VAL_SLOT = 1; + + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + static void finalize(JS::GCContext* gcx, JSObject* obj); + static void trace(JSTracer* trc, JSObject* obj); + + static bool typeImpl(JSContext* cx, const CallArgs& args); + static bool type(JSContext* cx, unsigned argc, Value* vp); + + static bool valueGetterImpl(JSContext* cx, const CallArgs& args); + static bool valueGetter(JSContext* cx, unsigned argc, Value* vp); + static bool valueSetterImpl(JSContext* cx, const CallArgs& args); + static bool valueSetter(JSContext* cx, unsigned argc, Value* vp); + + public: + static const unsigned RESERVED_SLOTS = 2; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + static WasmGlobalObject* create(JSContext* cx, wasm::HandleVal value, + bool isMutable, HandleObject proto); + bool isNewborn() { return getReservedSlot(VAL_SLOT).isUndefined(); } + + bool isMutable() const; + wasm::ValType type() const; + wasm::GCPtrVal& val() const; +}; + +// The class of WebAssembly.Instance. Each WasmInstanceObject owns a +// wasm::Instance. These objects are used both as content-facing JS objects and +// as internal implementation details of asm.js. + +class WasmInstanceObject : public NativeObject { + static const unsigned INSTANCE_SLOT = 0; + static const unsigned EXPORTS_OBJ_SLOT = 1; + static const unsigned EXPORTS_SLOT = 2; + static const unsigned SCOPES_SLOT = 3; + static const unsigned INSTANCE_SCOPE_SLOT = 4; + static const unsigned GLOBALS_SLOT = 5; + + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + static bool exportsGetterImpl(JSContext* cx, const CallArgs& args); + static bool exportsGetter(JSContext* cx, unsigned argc, Value* vp); + bool isNewborn() const; + static void finalize(JS::GCContext* gcx, JSObject* obj); + static void trace(JSTracer* trc, JSObject* obj); + + // ExportMap maps from function index to exported function object. + // This allows the instance to lazily create exported function + // objects on demand (instead up-front for all table elements) while + // correctly preserving observable function object identity. + using ExportMap = GCHashMap, + DefaultHasher, CellAllocPolicy>; + ExportMap& exports() const; + + // See the definition inside WasmJS.cpp. + class UnspecifiedScopeMap; + UnspecifiedScopeMap& scopes() const; + + public: + static const unsigned RESERVED_SLOTS = 6; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + static WasmInstanceObject* create( + JSContext* cx, const RefPtr& code, + const wasm::DataSegmentVector& dataSegments, + const wasm::ElemSegmentVector& elemSegments, uint32_t instanceDataLength, + Handle memory, + Vector, 0, SystemAllocPolicy>&& tables, + const JSObjectVector& funcImports, const wasm::GlobalDescVector& globals, + const wasm::ValVector& globalImportValues, + const WasmGlobalObjectVector& globalObjs, + const WasmTagObjectVector& tagObjs, HandleObject proto, + UniquePtr maybeDebug); + void initExportsObj(JSObject& exportsObj); + + wasm::Instance& instance() const; + JSObject& exportsObj() const; + + [[nodiscard]] static bool getExportedFunction( + JSContext* cx, Handle instanceObj, + uint32_t funcIndex, MutableHandleFunction fun); + + const wasm::CodeRange& getExportedFunctionCodeRange(JSFunction* fun, + wasm::Tier tier); + + static WasmInstanceScope* getScope(JSContext* cx, + Handle instanceObj); + static WasmFunctionScope* getFunctionScope( + JSContext* cx, Handle instanceObj, + uint32_t funcIndex); + + using GlobalObjectVector = + GCVector, 0, CellAllocPolicy>; + GlobalObjectVector& indirectGlobals() const; +}; + +// The class of WebAssembly.Memory. A WasmMemoryObject references an ArrayBuffer +// or SharedArrayBuffer object which owns the actual memory. + +class WasmMemoryObject : public NativeObject { + static const unsigned BUFFER_SLOT = 0; + static const unsigned OBSERVERS_SLOT = 1; + static const unsigned ISHUGE_SLOT = 2; + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + static void finalize(JS::GCContext* gcx, JSObject* obj); + static bool bufferGetterImpl(JSContext* cx, const CallArgs& args); + static bool bufferGetter(JSContext* cx, unsigned argc, Value* vp); + static bool typeImpl(JSContext* cx, const CallArgs& args); + static bool type(JSContext* cx, unsigned argc, Value* vp); + static bool growImpl(JSContext* cx, const CallArgs& args); + static bool grow(JSContext* cx, unsigned argc, Value* vp); + static bool discardImpl(JSContext* cx, const CallArgs& args); + static bool discard(JSContext* cx, unsigned argc, Value* vp); + static uint64_t growShared(Handle memory, uint64_t delta); + + using InstanceSet = JS::WeakCache, + StableCellHasher>, CellAllocPolicy>>; + bool hasObservers() const; + InstanceSet& observers() const; + InstanceSet* getOrCreateObservers(JSContext* cx); + + public: + static const unsigned RESERVED_SLOTS = 3; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec memoryControlMethods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + static WasmMemoryObject* create(JSContext* cx, + Handle buffer, + bool isHuge, HandleObject proto); + + // `buffer()` returns the current buffer object always. If the buffer + // represents shared memory then `buffer().byteLength()` never changes, and + // in particular it may be a smaller value than that returned from + // `volatileMemoryLength()` below. + // + // Generally, you do not want to call `buffer().byteLength()`, but to call + // `volatileMemoryLength()`, instead. + ArrayBufferObjectMaybeShared& buffer() const; + + // The current length of the memory in bytes. In the case of shared memory, + // the length can change at any time. Also note that this will acquire a lock + // for shared memory, so do not call this from a signal handler. + size_t volatileMemoryLength() const; + + // The current length of the memory in pages. See the comment for + // `volatileMemoryLength` for details on why this is 'volatile'. + wasm::Pages volatilePages() const; + + // The maximum length of the memory in pages. This is not 'volatile' in + // contrast to the current length, as it cannot change for shared memories. + wasm::Pages clampedMaxPages() const; + mozilla::Maybe sourceMaxPages() const; + + wasm::IndexType indexType() const; + bool isShared() const; + bool isHuge() const; + bool movingGrowable() const; + size_t boundsCheckLimit() const; + + // If isShared() is true then obtain the underlying buffer object. + WasmSharedArrayRawBuffer* sharedArrayRawBuffer() const; + + bool addMovingGrowObserver(JSContext* cx, WasmInstanceObject* instance); + static uint64_t grow(Handle memory, uint64_t delta, + JSContext* cx); + static void discard(Handle memory, uint64_t byteOffset, + uint64_t len, JSContext* cx); +}; + +// The class of WebAssembly.Table. A WasmTableObject holds a refcount on a +// wasm::Table, allowing a Table to be shared between multiple Instances +// (eventually between multiple threads). + +class WasmTableObject : public NativeObject { + static const unsigned TABLE_SLOT = 0; + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + bool isNewborn() const; + static void finalize(JS::GCContext* gcx, JSObject* obj); + static void trace(JSTracer* trc, JSObject* obj); + static bool lengthGetterImpl(JSContext* cx, const CallArgs& args); + static bool lengthGetter(JSContext* cx, unsigned argc, Value* vp); + static bool typeImpl(JSContext* cx, const CallArgs& args); + static bool type(JSContext* cx, unsigned argc, Value* vp); + static bool getImpl(JSContext* cx, const CallArgs& args); + static bool get(JSContext* cx, unsigned argc, Value* vp); + static bool setImpl(JSContext* cx, const CallArgs& args); + static bool set(JSContext* cx, unsigned argc, Value* vp); + static bool growImpl(JSContext* cx, const CallArgs& args); + static bool grow(JSContext* cx, unsigned argc, Value* vp); + + public: + static const unsigned RESERVED_SLOTS = 1; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + // Note that, after creation, a WasmTableObject's table() is not initialized + // and must be initialized before use. + + static WasmTableObject* create(JSContext* cx, uint32_t initialLength, + mozilla::Maybe maximumLength, + wasm::RefType tableType, HandleObject proto); + wasm::Table& table() const; + + // Perform the standard `ToWebAssemblyValue` coercion on `value` and fill the + // range [index, index + length) in the table. Callers are required to ensure + // the range is within bounds. Returns false if the coercion failed. + bool fillRange(JSContext* cx, uint32_t index, uint32_t length, + HandleValue value) const; +}; + +// The class of WebAssembly.Tag. This class is used to track exception tag +// types for exports and imports. + +class WasmTagObject : public NativeObject { + static const unsigned TYPE_SLOT = 0; + + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + static void finalize(JS::GCContext* gcx, JSObject* obj); + static bool typeImpl(JSContext* cx, const CallArgs& args); + static bool type(JSContext* cx, unsigned argc, Value* vp); + + public: + static const unsigned RESERVED_SLOTS = 1; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + static WasmTagObject* create(JSContext* cx, + const wasm::SharedTagType& tagType, + HandleObject proto); + + const wasm::TagType* tagType() const; + const wasm::ValTypeVector& valueTypes() const; + wasm::ResultType resultType() const; +}; + +// The class of WebAssembly.Exception. This class is used for +// representing exceptions thrown from Wasm in JS. (it is also used as +// the internal representation for exceptions in Wasm) + +class WasmExceptionObject : public NativeObject { + static const unsigned TAG_SLOT = 0; + static const unsigned TYPE_SLOT = 1; + static const unsigned DATA_SLOT = 2; + static const unsigned STACK_SLOT = 3; + + static const JSClassOps classOps_; + static const ClassSpec classSpec_; + static void trace(JSTracer* trc, JSObject* obj); + static void finalize(JS::GCContext* gcx, JSObject* obj); + // Named isMethod instead of is to avoid name conflict. + static bool isMethod(JSContext* cx, unsigned argc, Value* vp); + static bool isImpl(JSContext* cx, const CallArgs& args); + static bool getArg(JSContext* cx, unsigned argc, Value* vp); + static bool getArgImpl(JSContext* cx, const CallArgs& args); + static bool getStack(JSContext* cx, unsigned argc, Value* vp); + static bool getStack_impl(JSContext* cx, const CallArgs& args); + + uint8_t* typedMem() const; + [[nodiscard]] bool loadValue(JSContext* cx, size_t offset, wasm::ValType type, + MutableHandleValue vp); + [[nodiscard]] bool initValue(JSContext* cx, size_t offset, wasm::ValType type, + HandleValue value); + + public: + static const unsigned RESERVED_SLOTS = 4; + static const JSClass class_; + static const JSClass& protoClass_; + static const JSPropertySpec properties[]; + static const JSFunctionSpec methods[]; + static const JSFunctionSpec static_methods[]; + static bool construct(JSContext*, unsigned, Value*); + + static WasmExceptionObject* create(JSContext* cx, Handle tag, + HandleObject stack, HandleObject proto); + bool isNewborn() const; + + JSObject* stack() const; + const wasm::TagType* tagType() const; + WasmTagObject& tag() const; + + static size_t offsetOfData() { + return NativeObject::getFixedSlotOffset(DATA_SLOT); + } +}; + +// The class of the WebAssembly global namespace object. + +class WasmNamespaceObject : public NativeObject { + public: + static const JSClass class_; + + private: + static const ClassSpec classSpec_; +}; + +extern const JSClass WasmFunctionClass; + +} // namespace js + +#endif // wasm_js_h diff --git a/js/src/wasm/WasmLog.cpp b/js/src/wasm/WasmLog.cpp new file mode 100644 index 0000000000..34ef219ce1 --- /dev/null +++ b/js/src/wasm/WasmLog.cpp @@ -0,0 +1,81 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmLog.h" + +#include + +#include "jit/JitOptions.h" +#include "js/Printf.h" +#include "js/Utility.h" +#include "vm/JSContext.h" +#include "vm/Warnings.h" + +using namespace js; +using namespace js::wasm; + +void wasm::Log(JSContext* cx, const char* fmt, ...) { + MOZ_ASSERT(!cx->isExceptionPending()); + + if (!cx->options().wasmVerbose()) { + return; + } + + va_list args; + va_start(args, fmt); + + if (UniqueChars chars = JS_vsmprintf(fmt, args)) { + WarnNumberASCII(cx, JSMSG_WASM_VERBOSE, chars.get()); + if (cx->isExceptionPending()) { + cx->clearPendingException(); + } + } + + va_end(args); +} + +void wasm::LogOffThread(const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); +} + +#ifdef WASM_CODEGEN_DEBUG +bool wasm::IsCodegenDebugEnabled(DebugChannel channel) { + switch (channel) { + case DebugChannel::Function: + return jit::JitOptions.enableWasmFuncCallSpew; + case DebugChannel::Import: + return jit::JitOptions.enableWasmImportCallSpew; + } + return false; +} +#endif + +void wasm::DebugCodegen(DebugChannel channel, const char* fmt, ...) { +#ifdef WASM_CODEGEN_DEBUG + if (!IsCodegenDebugEnabled(channel)) { + return; + } + va_list ap; + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); +#endif +} diff --git a/js/src/wasm/WasmLog.h b/js/src/wasm/WasmLog.h new file mode 100644 index 0000000000..7ee5a5fc20 --- /dev/null +++ b/js/src/wasm/WasmLog.h @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_log_h +#define wasm_log_h + +#include "js/TypeDecls.h" + +namespace js { +namespace wasm { + +// Verbose logging support. + +extern void Log(JSContext* cx, const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3); +extern void LogOffThread(const char* fmt, ...) MOZ_FORMAT_PRINTF(1, 2); + +// Codegen debug support. + +enum class DebugChannel { + Function, + Import, +}; + +#ifdef WASM_CODEGEN_DEBUG +bool IsCodegenDebugEnabled(DebugChannel channel); +#endif + +void DebugCodegen(DebugChannel channel, const char* fmt, ...) + MOZ_FORMAT_PRINTF(2, 3); + +using PrintCallback = void (*)(const char*); + +} // namespace wasm +} // namespace js + +#endif // wasm_log_h diff --git a/js/src/wasm/WasmMemory.cpp b/js/src/wasm/WasmMemory.cpp new file mode 100644 index 0000000000..b5566921e0 --- /dev/null +++ b/js/src/wasm/WasmMemory.cpp @@ -0,0 +1,385 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmMemory.h" + +#include "mozilla/MathAlgorithms.h" + +#include "js/Conversions.h" +#include "js/ErrorReport.h" +#include "vm/ArrayBufferObject.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmProcess.h" + +using mozilla::IsPowerOfTwo; + +using namespace js; +using namespace js::wasm; + +const char* wasm::ToString(IndexType indexType) { + switch (indexType) { + case IndexType::I32: + return "i32"; + case IndexType::I64: + return "i64"; + default: + MOZ_CRASH(); + } +} + +bool wasm::ToIndexType(JSContext* cx, HandleValue value, IndexType* indexType) { + RootedString typeStr(cx, ToString(cx, value)); + if (!typeStr) { + return false; + } + + Rooted typeLinearStr(cx, typeStr->ensureLinear(cx)); + if (!typeLinearStr) { + return false; + } + + if (StringEqualsLiteral(typeLinearStr, "i32")) { + *indexType = IndexType::I32; + } else if (StringEqualsLiteral(typeLinearStr, "i64")) { + *indexType = IndexType::I64; + } else { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_STRING_IDX_TYPE); + return false; + } + return true; +} + +/* + * [SMDOC] Linear memory addresses and bounds checking + * + * (Also see "WASM Linear Memory structure" in vm/ArrayBufferObject.cpp) + * + * + * Memory addresses, bounds check avoidance, and the huge memory trick. + * + * A memory address in an access instruction has three components, the "memory + * base", the "pointer", and the "offset". The "memory base" - the HeapReg on + * most platforms and a value loaded from the instance on x86 - is a native + * pointer that points to the start of the linear memory array; we'll ignore the + * memory base in the following. The "pointer" is the i32 or i64 index supplied + * by the program as a separate value argument to the access instruction; it is + * usually variable but can be constant. The "offset" is a constant encoded in + * the access instruction. + * + * The "effective address" (EA) is the non-overflowed sum of the pointer and the + * offset (if the sum overflows the program traps); the pointer, offset, and EA + * all have the same type, i32 or i64. + * + * An access has an "access size", which is the number of bytes that are + * accessed - currently up to 16 (for V128). The highest-addressed byte to be + * accessed by an access is thus the byte at (pointer+offset+access_size-1), + * where offset+access_size-1 is compile-time evaluable. + * + * Bounds checking ensures that the entire access is in bounds, ie, that the + * highest-addressed byte is at an offset in the linear memory below that of the + * memory's current byteLength. + * + * To avoid performing an addition with overflow check and a compare-and-branch + * bounds check for every memory access, we use some tricks: + * + * - An access-protected guard region of size R at the end of each memory is + * used to trap accesses to out-of-bounds offsets in the range + * 0..R-access_size. Thus the offset and the access size need not be added + * into the pointer before the bounds check, saving the add and overflow + * check. The offset is added into the pointer without an overflow check + * either directly before the access or in the access instruction itself + * (depending on the ISA). The pointer must still be explicitly + * bounds-checked. + * + * - On 64-bit systems where we determine there is plenty of virtual memory + * space (and ideally we determine that the VM system uses overcommit), a + * 32-bit memory is implemented as a 4GB + R reservation, where the memory + * from the current heap length through the end of the reservation is + * access-protected. The protected area R allows offsets up to R-access_size + * to be encoded in the access instruction. The pointer need not be bounds + * checked explicitly, since it has only a 4GB range and thus points into the + * 4GB part of the reservation. The offset can be added into the pointer + * (using 64-bit arithmetic) either directly before the access or in the + * access instruction. + * + * The value of R differs in the two situations; in the first case it tends to + * be small, currently 64KB; in the second case it is large, currently 2GB+64KB. + * The difference is due to explicit bounds checking tending to be used on + * 32-bit systems where memory and address space are scarce, while the implicit + * bounds check is used only on 64-bit systems after ensuring that sufficient + * address space is available in the process. (2GB is really overkill, and + * there's nothing magic about it; we could use something much smaller.) + * + * The implicit bounds checking strategy with the large reservation is known + * below and elsewhere as the "huge memory trick" or just "huge memory". + * + * All memories in a process use the same strategy, selected at process startup. + * The immediate reason for that is that the machine code embeds the strategy + * it's been compiled with, and may later be exposed to memories originating + * from different modules or directly from JS. If the memories did not all use + * the same strategy, and the same strategy as the code, linking would fail or + * we would have to recompile the code. + * + * + * The boundsCheckLimit. + * + * The bounds check limit that is stored in the instance is always valid and is + * always a 64-bit datum, and it is always correct to load it and use it as a + * 64-bit value. However, in situations when the 32 upper bits are known to be + * zero, it is also correct to load just the low 32 bits from the address of the + * limit (which is always little-endian when a JIT is enabled), and use that + * value as the limit. + * + * On x86 and arm32 (and on any other 32-bit platform, should there ever be + * one), there is explicit bounds checking and the heap, whether memory32 or + * memory64, is limited to 2GB; the bounds check limit can be treated as a + * 32-bit quantity. + * + * On all 64-bit platforms, we may use explicit bounds checking or the huge + * memory trick for memory32, but must always use explicit bounds checking for + * memory64. If the heap does not have a known maximum size or the known + * maximum is greater than or equal to 4GB, then the bounds check limit must be + * treated as a 64-bit quantity; otherwise it can be treated as a 32-bit + * quantity. + * + * On x64 and arm64 with Baseline and Ion, we allow 32-bit memories up to 4GB, + * and 64-bit memories can be larger. + * + * On mips64, memories are limited to 2GB, for now. + * + * Asm.js memories are limited to 2GB even on 64-bit platforms, and we can + * always assume a 32-bit bounds check limit for asm.js. + * + * + * Constant pointers. + * + * If the pointer is constant then the EA can be computed at compile time, and + * if the EA is below the initial memory size then the bounds check can be + * elided. + * + * + * Alignment checks. + * + * On all platforms, some accesses (currently atomics) require an alignment + * check: the EA must be naturally aligned for the datum being accessed. + * However, we do not need to compute the EA properly, we care only about the + * low bits - a cheap, overflowing add is fine, and if the offset is known + * to be aligned, only the pointer need be checked. + */ + +// Bounds checks always compare the base of the memory access with the bounds +// check limit. If the memory access is unaligned, this means that, even if the +// bounds check succeeds, a few bytes of the access can extend past the end of +// memory. To guard against this, extra space is included in the guard region to +// catch the overflow. MaxMemoryAccessSize is a conservative approximation of +// the maximum guard space needed to catch all unaligned overflows. +// +// Also see "Linear memory addresses and bounds checking" above. + +static const unsigned MaxMemoryAccessSize = LitVal::sizeofLargestValue(); + +// All plausible targets must be able to do at least IEEE754 double +// loads/stores, hence the lower limit of 8. Some Intel processors support +// AVX-512 loads/stores, hence the upper limit of 64. +static_assert(MaxMemoryAccessSize >= 8, "MaxMemoryAccessSize too low"); +static_assert(MaxMemoryAccessSize <= 64, "MaxMemoryAccessSize too high"); +static_assert((MaxMemoryAccessSize & (MaxMemoryAccessSize - 1)) == 0, + "MaxMemoryAccessSize is not a power of two"); + +#ifdef WASM_SUPPORTS_HUGE_MEMORY + +static_assert(MaxMemoryAccessSize <= HugeUnalignedGuardPage, + "rounded up to static page size"); +static_assert(HugeOffsetGuardLimit < UINT32_MAX, + "checking for overflow against OffsetGuardLimit is enough."); + +// We have only tested huge memory on x64 and arm64. +# if !(defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64)) +# error "Not an expected configuration" +# endif + +#endif + +// On !WASM_SUPPORTS_HUGE_MEMORY platforms: +// - To avoid OOM in ArrayBuffer::prepareForAsmJS, asm.js continues to use the +// original ArrayBuffer allocation which has no guard region at all. +// - For WebAssembly memories, an additional GuardSize is mapped after the +// accessible region of the memory to catch folded (base+offset) accesses +// where `offset < OffsetGuardLimit` as well as the overflow from unaligned +// accesses, as described above for MaxMemoryAccessSize. + +static const size_t OffsetGuardLimit = PageSize - MaxMemoryAccessSize; + +static_assert(MaxMemoryAccessSize < GuardSize, + "Guard page handles partial out-of-bounds"); +static_assert(OffsetGuardLimit < UINT32_MAX, + "checking for overflow against OffsetGuardLimit is enough."); + +size_t wasm::GetMaxOffsetGuardLimit(bool hugeMemory) { +#ifdef WASM_SUPPORTS_HUGE_MEMORY + return hugeMemory ? HugeOffsetGuardLimit : OffsetGuardLimit; +#else + return OffsetGuardLimit; +#endif +} + +// Assert that our minimum offset guard limit covers our inline +// memory.copy/fill optimizations. +static const size_t MinOffsetGuardLimit = OffsetGuardLimit; +static_assert(MaxInlineMemoryCopyLength < MinOffsetGuardLimit, "precondition"); +static_assert(MaxInlineMemoryFillLength < MinOffsetGuardLimit, "precondition"); + +#ifdef JS_64BIT +wasm::Pages wasm::MaxMemoryPages(IndexType t) { + MOZ_ASSERT_IF(t == IndexType::I64, !IsHugeMemoryEnabled(t)); + size_t desired = MaxMemoryLimitField(t); + constexpr size_t actual = ArrayBufferObject::MaxByteLength / PageSize; + return wasm::Pages(std::min(desired, actual)); +} + +size_t wasm::MaxMemoryBoundsCheckLimit(IndexType t) { + return MaxMemoryPages(t).byteLength(); +} + +#else +// On 32-bit systems, the heap limit must be representable in the nonnegative +// range of an int32_t, which means the maximum heap size as observed by wasm +// code is one wasm page less than 2GB. +wasm::Pages wasm::MaxMemoryPages(IndexType t) { + static_assert(ArrayBufferObject::MaxByteLength >= INT32_MAX / PageSize); + return wasm::Pages(INT32_MAX / PageSize); +} + +// The max bounds check limit can be larger than the MaxMemoryPages because it +// is really MaxMemoryPages rounded up to the next valid bounds check immediate, +// see ComputeMappedSize(). +size_t wasm::MaxMemoryBoundsCheckLimit(IndexType t) { + size_t boundsCheckLimit = size_t(INT32_MAX) + 1; + MOZ_ASSERT(IsValidBoundsCheckImmediate(boundsCheckLimit)); + return boundsCheckLimit; +} +#endif + +// Because ARM has a fixed-width instruction encoding, ARM can only express a +// limited subset of immediates (in a single instruction). + +static const uint64_t HighestValidARMImmediate = 0xff000000; + +// Heap length on ARM should fit in an ARM immediate. We approximate the set +// of valid ARM immediates with the predicate: +// 2^n for n in [16, 24) +// or +// 2^24 * n for n >= 1. +bool wasm::IsValidARMImmediate(uint32_t i) { + bool valid = (IsPowerOfTwo(i) || (i & 0x00ffffff) == 0); + + MOZ_ASSERT_IF(valid, i % PageSize == 0); + + return valid; +} + +uint64_t wasm::RoundUpToNextValidARMImmediate(uint64_t i) { + MOZ_ASSERT(i <= HighestValidARMImmediate); + static_assert(HighestValidARMImmediate == 0xff000000, + "algorithm relies on specific constant"); + + if (i <= 16 * 1024 * 1024) { + i = i ? mozilla::RoundUpPow2(i) : 0; + } else { + i = (i + 0x00ffffff) & ~0x00ffffff; + } + + MOZ_ASSERT(IsValidARMImmediate(i)); + + return i; +} + +Pages wasm::ClampedMaxPages(IndexType t, Pages initialPages, + const Maybe& sourceMaxPages, + bool useHugeMemory) { + Pages clampedMaxPages; + + if (sourceMaxPages.isSome()) { + // There is a specified maximum, clamp it to the implementation limit of + // maximum pages + clampedMaxPages = std::min(*sourceMaxPages, wasm::MaxMemoryPages(t)); + +#ifndef JS_64BIT + static_assert(sizeof(uintptr_t) == 4, "assuming not 64 bit implies 32 bit"); + + // On 32-bit platforms, prevent applications specifying a large max (like + // MaxMemoryPages()) from unintentially OOMing the browser: they just want + // "a lot of memory". Maintain the invariant that initialPages <= + // clampedMaxPages. + static const uint64_t OneGib = 1 << 30; + static const Pages OneGibPages = Pages(OneGib >> wasm::PageBits); + static_assert(HighestValidARMImmediate > OneGib, + "computing mapped size on ARM requires clamped max size"); + + Pages clampedPages = std::max(OneGibPages, initialPages); + clampedMaxPages = std::min(clampedPages, clampedMaxPages); +#endif + } else { + // There is not a specified maximum, fill it in with the implementation + // limit of maximum pages + clampedMaxPages = wasm::MaxMemoryPages(t); + } + + // Double-check our invariants + MOZ_RELEASE_ASSERT(sourceMaxPages.isNothing() || + clampedMaxPages <= *sourceMaxPages); + MOZ_RELEASE_ASSERT(clampedMaxPages <= wasm::MaxMemoryPages(t)); + MOZ_RELEASE_ASSERT(initialPages <= clampedMaxPages); + + return clampedMaxPages; +} + +size_t wasm::ComputeMappedSize(wasm::Pages clampedMaxPages) { + // Caller is responsible to ensure that clampedMaxPages has been clamped to + // implementation limits. + size_t maxSize = clampedMaxPages.byteLength(); + + // It is the bounds-check limit, not the mapped size, that gets baked into + // code. Thus round up the maxSize to the next valid immediate value + // *before* adding in the guard page. + // + // Also see "Wasm Linear Memory Structure" in vm/ArrayBufferObject.cpp. + uint64_t boundsCheckLimit = RoundUpToNextValidBoundsCheckImmediate(maxSize); + MOZ_ASSERT(IsValidBoundsCheckImmediate(boundsCheckLimit)); + + MOZ_ASSERT(boundsCheckLimit % gc::SystemPageSize() == 0); + MOZ_ASSERT(GuardSize % gc::SystemPageSize() == 0); + return boundsCheckLimit + GuardSize; +} + +bool wasm::IsValidBoundsCheckImmediate(uint32_t i) { +#ifdef JS_CODEGEN_ARM + return IsValidARMImmediate(i); +#else + return true; +#endif +} + +uint64_t wasm::RoundUpToNextValidBoundsCheckImmediate(uint64_t i) { +#ifdef JS_CODEGEN_ARM + return RoundUpToNextValidARMImmediate(i); +#else + return i; +#endif +} diff --git a/js/src/wasm/WasmMemory.h b/js/src/wasm/WasmMemory.h new file mode 100644 index 0000000000..ea2c61aa38 --- /dev/null +++ b/js/src/wasm/WasmMemory.h @@ -0,0 +1,226 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_memory_h +#define wasm_memory_h + +#include "mozilla/CheckedInt.h" +#include "mozilla/Maybe.h" + +#include + +#include "js/Value.h" +#include "vm/NativeObject.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmValType.h" + +namespace js { +namespace wasm { + +// Limits are parameterized by an IndexType which is used to index the +// underlying resource (either a Memory or a Table). Tables are restricted to +// I32, while memories may use I64 when memory64 is enabled. + +enum class IndexType : uint8_t { I32, I64 }; + +inline ValType ToValType(IndexType it) { + return it == IndexType::I64 ? ValType::I64 : ValType::I32; +} + +extern bool ToIndexType(JSContext* cx, HandleValue value, IndexType* indexType); + +extern const char* ToString(IndexType indexType); + +// Pages is a typed unit representing a multiple of wasm::PageSize. We +// generally use pages as the unit of length when representing linear memory +// lengths so as to avoid overflow when the specified initial or maximum pages +// would overflow the native word size. +// +// Modules may specify pages up to 2^48 inclusive and so Pages is 64-bit on all +// platforms. +// +// We represent byte lengths using the native word size, as it is assumed that +// consumers of this API will only need byte lengths once it is time to +// allocate memory, at which point the pages will be checked against the +// implementation limits `MaxMemoryPages()` and will then be guaranteed to +// fit in a native word. +struct Pages { + private: + // Pages are specified by limit fields, which in general may be up to 2^48, + // so we must use uint64_t here. + uint64_t value_; + + public: + constexpr Pages() : value_(0) {} + constexpr explicit Pages(uint64_t value) : value_(value) {} + + // Get the wrapped page value. Only use this if you must, prefer to use or + // add new APIs to Page. + uint64_t value() const { return value_; } + + // Converts from a byte length to pages, assuming that the length is an + // exact multiple of the page size. + static Pages fromByteLengthExact(size_t byteLength) { + MOZ_ASSERT(byteLength % PageSize == 0); + return Pages(byteLength / PageSize); + } + + // Return whether the page length may overflow when converted to a byte + // length in the native word size. + bool hasByteLength() const { + mozilla::CheckedInt length(value_); + length *= PageSize; + return length.isValid(); + } + + // Converts from pages to byte length in the native word size. Users must + // check for overflow, or be assured else-how that overflow cannot happen. + size_t byteLength() const { + mozilla::CheckedInt length(value_); + length *= PageSize; + return length.value(); + } + + // Increment this pages by delta and return whether the resulting value + // did not overflow. If there is no overflow, then this is set to the + // resulting value. + bool checkedIncrement(Pages delta) { + mozilla::CheckedInt newValue = value_; + newValue += delta.value_; + if (!newValue.isValid()) { + return false; + } + value_ = newValue.value(); + return true; + } + + // Implement pass-through comparison operators so that Pages can be compared. + + bool operator==(Pages other) const { return value_ == other.value_; } + bool operator!=(Pages other) const { return value_ != other.value_; } + bool operator<=(Pages other) const { return value_ <= other.value_; } + bool operator<(Pages other) const { return value_ < other.value_; } + bool operator>=(Pages other) const { return value_ >= other.value_; } + bool operator>(Pages other) const { return value_ > other.value_; } +}; + +// The largest number of pages the application can request. +extern Pages MaxMemoryPages(IndexType t); + +// The byte value of MaxMemoryPages(t). +static inline size_t MaxMemoryBytes(IndexType t) { + return MaxMemoryPages(t).byteLength(); +} + +// A value at least as large as MaxMemoryBytes(t) representing the largest valid +// bounds check limit on the system. (It can be larger than MaxMemoryBytes() +// because bounds check limits are rounded up to fit formal requirements on some +// platforms. Also see ComputeMappedSize().) +extern size_t MaxMemoryBoundsCheckLimit(IndexType t); + +static inline uint64_t MaxMemoryLimitField(IndexType indexType) { + return indexType == IndexType::I32 ? MaxMemory32LimitField + : MaxMemory64LimitField; +} + +// Compute the 'clamped' maximum size of a memory. See +// 'WASM Linear Memory structure' in ArrayBufferObject.cpp for background. +extern Pages ClampedMaxPages(IndexType t, Pages initialPages, + const mozilla::Maybe& sourceMaxPages, + bool useHugeMemory); + +// For a given WebAssembly/asm.js 'clamped' max pages, return the number of +// bytes to map which will necessarily be a multiple of the system page size and +// greater than clampedMaxPages in bytes. See "Wasm Linear Memory Structure" in +// vm/ArrayBufferObject.cpp. +extern size_t ComputeMappedSize(Pages clampedMaxPages); + +extern size_t GetMaxOffsetGuardLimit(bool hugeMemory); + +// Return whether the given immediate satisfies the constraints of the platform. +extern bool IsValidBoundsCheckImmediate(uint32_t i); + +// Return whether the given immediate is valid on arm. +extern bool IsValidARMImmediate(uint32_t i); + +// Return the next higher valid immediate that satisfies the constraints of the +// platform. +extern uint64_t RoundUpToNextValidBoundsCheckImmediate(uint64_t i); + +// Return the next higher valid immediate for arm. +extern uint64_t RoundUpToNextValidARMImmediate(uint64_t i); + +#ifdef WASM_SUPPORTS_HUGE_MEMORY +// On WASM_SUPPORTS_HUGE_MEMORY platforms, every asm.js or WebAssembly 32-bit +// memory unconditionally allocates a huge region of virtual memory of size +// wasm::HugeMappedSize. This allows all memory resizing to work without +// reallocation and provides enough guard space for most offsets to be folded +// into memory accesses. See "Linear memory addresses and bounds checking" in +// wasm/WasmMemory.cpp for more information. + +// Reserve 4GiB to support any i32 index. +static const uint64_t HugeIndexRange = uint64_t(UINT32_MAX) + 1; +// Reserve 32MiB to support most offset immediates. Any immediate that is over +// this will require a bounds check to be emitted. 32MiB was chosen to +// generously cover the max offset immediate, 20MiB, found in a corpus of wasm +// modules. +static const uint64_t HugeOffsetGuardLimit = 1 << 25; +// Reserve a wasm page (64KiB) to support slop on unaligned accesses. +static const uint64_t HugeUnalignedGuardPage = PageSize; + +// Compute the total memory reservation. +static const uint64_t HugeMappedSize = + HugeIndexRange + HugeOffsetGuardLimit + HugeUnalignedGuardPage; + +// Try to keep the memory reservation aligned to the wasm page size. This +// ensures that it's aligned to the system page size. +static_assert(HugeMappedSize % PageSize == 0); + +#endif + +// The size of the guard page for non huge-memories. +static const size_t GuardSize = PageSize; + +// The size of the guard page that included NULL pointer. Reserve a smallest +// range for typical hardware, to catch near NULL pointer accesses, e.g. +// for a structure fields operations. +static const size_t NullPtrGuardSize = 4096; + +// Check if a range of wasm memory is within bounds, specified as byte offset +// and length (using 32-bit indices). Omits one check by converting from +// uint32_t to uint64_t, at which point overflow cannot occur. +static inline bool MemoryBoundsCheck(uint32_t offset, uint32_t len, + size_t memLen) { + uint64_t offsetLimit = uint64_t(offset) + uint64_t(len); + return offsetLimit <= memLen; +} + +// Check if a range of wasm memory is within bounds, specified as byte offset +// and length (using 64-bit indices). +static inline bool MemoryBoundsCheck(uint64_t offset, uint64_t len, + size_t memLen) { + uint64_t offsetLimit = offset + len; + bool didOverflow = offsetLimit < offset; + bool tooLong = memLen < offsetLimit; + return !didOverflow && !tooLong; +} + +} // namespace wasm +} // namespace js + +#endif // wasm_memory_h diff --git a/js/src/wasm/WasmModule.cpp b/js/src/wasm/WasmModule.cpp new file mode 100644 index 0000000000..b30c810e0f --- /dev/null +++ b/js/src/wasm/WasmModule.cpp @@ -0,0 +1,1134 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmModule.h" + +#include + +#include "jit/FlushICache.h" // for FlushExecutionContextForAllThreads +#include "js/BuildId.h" // JS::BuildIdCharVector +#include "js/experimental/TypedData.h" // JS_NewUint8Array +#include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_* +#include "js/Printf.h" // JS_smprintf +#include "js/PropertyAndElement.h" // JS_DefineProperty, JS_DefinePropertyById +#include "js/StreamConsumer.h" +#include "threading/LockGuard.h" +#include "threading/Thread.h" +#include "vm/HelperThreadState.h" // Tier2GeneratorTask +#include "vm/PlainObject.h" // js::PlainObject +#include "wasm/WasmBaselineCompile.h" +#include "wasm/WasmCompile.h" +#include "wasm/WasmDebug.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmIonCompile.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmUtility.h" + +#include "debugger/DebugAPI-inl.h" +#include "vm/ArrayBufferObject-inl.h" +#include "vm/JSAtom-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +static UniqueChars Tier2ResultsContext(const ScriptedCaller& scriptedCaller) { + return scriptedCaller.filename + ? JS_smprintf("%s:%d", scriptedCaller.filename.get(), + scriptedCaller.line) + : UniqueChars(); +} + +static void ReportTier2ResultsOffThread(bool success, + const ScriptedCaller& scriptedCaller, + const UniqueChars& error, + const UniqueCharsVector& warnings) { + // Get context to describe this tier-2 task. + UniqueChars context = Tier2ResultsContext(scriptedCaller); + const char* contextString = context ? context.get() : "unknown"; + + // Display the main error, if any. + if (!success) { + const char* errorString = error ? error.get() : "out of memory"; + LogOffThread("'%s': wasm tier-2 failed with '%s'.\n", contextString, + errorString); + } + + // Display warnings as a follow-up, avoiding spamming the console. + size_t numWarnings = std::min(warnings.length(), 3); + + for (size_t i = 0; i < numWarnings; i++) { + LogOffThread("'%s': wasm tier-2 warning: '%s'.\n'.", contextString, + warnings[i].get()); + } + if (warnings.length() > numWarnings) { + LogOffThread("'%s': other warnings suppressed.\n", contextString); + } +} + +class Module::Tier2GeneratorTaskImpl : public Tier2GeneratorTask { + SharedCompileArgs compileArgs_; + SharedBytes bytecode_; + SharedModule module_; + Atomic cancelled_; + + public: + Tier2GeneratorTaskImpl(const CompileArgs& compileArgs, + const ShareableBytes& bytecode, Module& module) + : compileArgs_(&compileArgs), + bytecode_(&bytecode), + module_(&module), + cancelled_(false) {} + + ~Tier2GeneratorTaskImpl() override { + module_->tier2Listener_ = nullptr; + module_->testingTier2Active_ = false; + } + + void cancel() override { cancelled_ = true; } + + void runHelperThreadTask(AutoLockHelperThreadState& locked) override { + { + AutoUnlockHelperThreadState unlock(locked); + + // Compile tier-2 and report any warning/errors as long as it's not a + // cancellation. Encountering a warning/error during compilation and + // being cancelled may race with each other, but the only observable race + // should be being cancelled after a warning/error is set, and that's + // okay. + UniqueChars error; + UniqueCharsVector warnings; + bool success = CompileTier2(*compileArgs_, bytecode_->bytes, *module_, + &error, &warnings, &cancelled_); + if (!cancelled_) { + // We could try to dispatch a runnable to the thread that started this + // compilation, so as to report the warning/error using a JSContext*. + // For now we just report to stderr. + ReportTier2ResultsOffThread(success, compileArgs_->scriptedCaller, + error, warnings); + } + } + + // During shutdown the main thread will wait for any ongoing (cancelled) + // tier-2 generation to shut down normally. To do so, it waits on the + // HelperThreadState's condition variable for the count of finished + // generators to rise. + HelperThreadState().incWasmTier2GeneratorsFinished(locked); + + // The task is finished, release it. + js_delete(this); + } + + ThreadType threadType() override { + return ThreadType::THREAD_TYPE_WASM_GENERATOR_TIER2; + } +}; + +Module::~Module() { + // Note: Modules can be destroyed on any thread. + MOZ_ASSERT(!tier2Listener_); + MOZ_ASSERT(!testingTier2Active_); +} + +void Module::startTier2(const CompileArgs& args, const ShareableBytes& bytecode, + JS::OptimizedEncodingListener* listener) { + MOZ_ASSERT(!testingTier2Active_); + + auto task = MakeUnique(args, bytecode, *this); + if (!task) { + return; + } + + // These will be cleared asynchronously by ~Tier2GeneratorTaskImpl() if not + // sooner by finishTier2(). + tier2Listener_ = listener; + testingTier2Active_ = true; + + StartOffThreadWasmTier2Generator(std::move(task)); +} + +bool Module::finishTier2(const LinkData& linkData2, + UniqueCodeTier code2) const { + MOZ_ASSERT(code().bestTier() == Tier::Baseline && + code2->tier() == Tier::Optimized); + + // Install the data in the data structures. They will not be visible + // until commitTier2(). + + const CodeTier* borrowedTier2; + if (!code().setAndBorrowTier2(std::move(code2), linkData2, &borrowedTier2)) { + return false; + } + + // Before we can make tier-2 live, we need to compile tier2 versions of any + // extant tier1 lazy stubs (otherwise, tiering would break the assumption + // that any extant exported wasm function has had a lazy entry stub already + // compiled for it). + // + // Also see doc block for stubs in WasmJS.cpp. + { + // We need to prevent new tier1 stubs generation until we've committed + // the newer tier2 stubs, otherwise we might not generate one tier2 + // stub that has been generated for tier1 before we committed. + + const MetadataTier& metadataTier1 = metadata(Tier::Baseline); + + auto stubs1 = code().codeTier(Tier::Baseline).lazyStubs().readLock(); + auto stubs2 = borrowedTier2->lazyStubs().writeLock(); + + MOZ_ASSERT(stubs2->entryStubsEmpty()); + + Uint32Vector funcExportIndices; + for (size_t i = 0; i < metadataTier1.funcExports.length(); i++) { + const FuncExport& fe = metadataTier1.funcExports[i]; + if (fe.hasEagerStubs()) { + continue; + } + if (!stubs1->hasEntryStub(fe.funcIndex())) { + continue; + } + if (!funcExportIndices.emplaceBack(i)) { + return false; + } + } + + Maybe stub2Index; + if (!stubs2->createTier2(funcExportIndices, metadata(), *borrowedTier2, + &stub2Index)) { + return false; + } + + // Initializing the code above will have flushed the icache for all cores. + // However, there could still be stale data in the execution pipeline of + // other cores on some platforms. Force an execution context flush on all + // threads to fix this before we commit the code. + // + // This is safe due to the check in `PlatformCanTier` in WasmCompile.cpp + jit::FlushExecutionContextForAllThreads(); + + // Now that we can't fail or otherwise abort tier2, make it live. + + MOZ_ASSERT(!code().hasTier2()); + code().commitTier2(); + + stubs2->setJitEntries(stub2Index, code()); + } + + // And we update the jump vectors with pointers to tier-2 functions and eager + // stubs. Callers will continue to invoke tier-1 code until, suddenly, they + // will invoke tier-2 code. This is benign. + + uint8_t* base = code().segment(Tier::Optimized).base(); + for (const CodeRange& cr : metadata(Tier::Optimized).codeRanges) { + // These are racy writes that we just want to be visible, atomically, + // eventually. All hardware we care about will do this right. But + // we depend on the compiler not splitting the stores hidden inside the + // set*Entry functions. + if (cr.isFunction()) { + code().setTieringEntry(cr.funcIndex(), base + cr.funcTierEntry()); + } else if (cr.isJitEntry()) { + code().setJitEntry(cr.funcIndex(), base + cr.begin()); + } + } + + // Tier-2 is done; let everyone know. Mark tier-2 active for testing + // purposes so that wasmHasTier2CompilationCompleted() only returns true + // after tier-2 has been fully cached. + + if (tier2Listener_) { + Bytes bytes; + if (serialize(linkData2, &bytes)) { + tier2Listener_->storeOptimizedEncoding(bytes.begin(), bytes.length()); + } + tier2Listener_ = nullptr; + } + testingTier2Active_ = false; + + return true; +} + +void Module::testingBlockOnTier2Complete() const { + while (testingTier2Active_) { + ThisThread::SleepMilliseconds(1); + } +} + +/* virtual */ +JSObject* Module::createObject(JSContext* cx) const { + if (!GlobalObject::ensureConstructor(cx, cx->global(), JSProto_WebAssembly)) { + return nullptr; + } + + if (!cx->isRuntimeCodeGenEnabled(JS::RuntimeCode::WASM, nullptr)) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_CSP_BLOCKED_WASM, "WebAssembly.Module"); + return nullptr; + } + + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmModule)); + return WasmModuleObject::create(cx, *this, proto); +} + +/* virtual */ +JSObject* Module::createObjectForAsmJS(JSContext* cx) const { + // Use nullptr to get the default object prototype. These objects are never + // exposed to script for asm.js. + return WasmModuleObject::create(cx, *this, nullptr); +} + +bool wasm::GetOptimizedEncodingBuildId(JS::BuildIdCharVector* buildId) { + // From a JS API perspective, the "build id" covers everything that can + // cause machine code to become invalid, so include both the actual build-id + // and cpu-id. + + if (!GetBuildId || !GetBuildId(buildId)) { + return false; + } + + uint32_t cpu = ObservedCPUFeatures(); + + if (!buildId->reserve(buildId->length() + + 13 /* "()" + 8 nibbles + "m[+-][+-]" */)) { + return false; + } + + buildId->infallibleAppend('('); + while (cpu) { + buildId->infallibleAppend('0' + (cpu & 0xf)); + cpu >>= 4; + } + buildId->infallibleAppend(')'); + + buildId->infallibleAppend('m'); + buildId->infallibleAppend(wasm::IsHugeMemoryEnabled(IndexType::I32) ? '+' + : '-'); + buildId->infallibleAppend(wasm::IsHugeMemoryEnabled(IndexType::I64) ? '+' + : '-'); + + return true; +} + +/* virtual */ +void Module::addSizeOfMisc(MallocSizeOf mallocSizeOf, + Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, size_t* code, + size_t* data) const { + code_->addSizeOfMiscIfNotSeen(mallocSizeOf, seenMetadata, seenCode, code, + data); + *data += mallocSizeOf(this) + + SizeOfVectorExcludingThis(imports_, mallocSizeOf) + + SizeOfVectorExcludingThis(exports_, mallocSizeOf) + + SizeOfVectorExcludingThis(dataSegments_, mallocSizeOf) + + SizeOfVectorExcludingThis(elemSegments_, mallocSizeOf) + + SizeOfVectorExcludingThis(customSections_, mallocSizeOf); +} + +// Extracting machine code as JS object. The result has the "code" property, as +// a Uint8Array, and the "segments" property as array objects. The objects +// contain offsets in the "code" array and basic information about a code +// segment/function body. +bool Module::extractCode(JSContext* cx, Tier tier, + MutableHandleValue vp) const { + Rooted result(cx, NewPlainObject(cx)); + if (!result) { + return false; + } + + // This function is only used for testing purposes so we can simply + // block on tiered compilation to complete. + testingBlockOnTier2Complete(); + + if (!code_->hasTier(tier)) { + vp.setNull(); + return true; + } + + const ModuleSegment& moduleSegment = code_->segment(tier); + RootedObject code(cx, JS_NewUint8Array(cx, moduleSegment.length())); + if (!code) { + return false; + } + + memcpy(code->as().dataPointerUnshared(), + moduleSegment.base(), moduleSegment.length()); + + RootedValue value(cx, ObjectValue(*code)); + if (!JS_DefineProperty(cx, result, "code", value, JSPROP_ENUMERATE)) { + return false; + } + + RootedObject segments(cx, NewDenseEmptyArray(cx)); + if (!segments) { + return false; + } + + for (const CodeRange& p : metadata(tier).codeRanges) { + RootedObject segment(cx, NewPlainObjectWithProto(cx, nullptr)); + if (!segment) { + return false; + } + + value.setNumber((uint32_t)p.begin()); + if (!JS_DefineProperty(cx, segment, "begin", value, JSPROP_ENUMERATE)) { + return false; + } + + value.setNumber((uint32_t)p.end()); + if (!JS_DefineProperty(cx, segment, "end", value, JSPROP_ENUMERATE)) { + return false; + } + + value.setNumber((uint32_t)p.kind()); + if (!JS_DefineProperty(cx, segment, "kind", value, JSPROP_ENUMERATE)) { + return false; + } + + if (p.isFunction()) { + value.setNumber((uint32_t)p.funcIndex()); + if (!JS_DefineProperty(cx, segment, "funcIndex", value, + JSPROP_ENUMERATE)) { + return false; + } + + value.setNumber((uint32_t)p.funcUncheckedCallEntry()); + if (!JS_DefineProperty(cx, segment, "funcBodyBegin", value, + JSPROP_ENUMERATE)) { + return false; + } + + value.setNumber((uint32_t)p.end()); + if (!JS_DefineProperty(cx, segment, "funcBodyEnd", value, + JSPROP_ENUMERATE)) { + return false; + } + } + + if (!NewbornArrayPush(cx, segments, ObjectValue(*segment))) { + return false; + } + } + + value.setObject(*segments); + if (!JS_DefineProperty(cx, result, "segments", value, JSPROP_ENUMERATE)) { + return false; + } + + vp.setObject(*result); + return true; +} + +#ifdef DEBUG +static bool AllSegmentsArePassive(const DataSegmentVector& vec) { + for (const DataSegment* seg : vec) { + if (seg->active()) { + return false; + } + } + return true; +} +#endif + +bool Module::initSegments(JSContext* cx, + Handle instanceObj, + Handle memoryObj) const { + MOZ_ASSERT_IF(!memoryObj, AllSegmentsArePassive(dataSegments_)); + + Instance& instance = instanceObj->instance(); + const SharedTableVector& tables = instance.tables(); + + // Write data/elem segments into memories/tables. + + for (const ElemSegment* seg : elemSegments_) { + if (seg->active()) { + RootedVal offsetVal(cx); + if (!seg->offset().evaluate(cx, instanceObj, &offsetVal)) { + return false; // OOM + } + uint32_t offset = offsetVal.get().i32(); + uint32_t count = seg->length(); + + uint32_t tableLength = tables[seg->tableIndex]->length(); + if (offset > tableLength || tableLength - offset < count) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_OUT_OF_BOUNDS); + return false; + } + + if (!instance.initElems(seg->tableIndex, *seg, offset, 0, count)) { + return false; // OOM + } + } + } + + if (memoryObj) { + size_t memoryLength = memoryObj->volatileMemoryLength(); + uint8_t* memoryBase = + memoryObj->buffer().dataPointerEither().unwrap(/* memcpy */); + + for (const DataSegment* seg : dataSegments_) { + if (!seg->active()) { + continue; + } + + RootedVal offsetVal(cx); + if (!seg->offset().evaluate(cx, instanceObj, &offsetVal)) { + return false; // OOM + } + uint64_t offset = memoryObj->indexType() == IndexType::I32 + ? offsetVal.get().i32() + : offsetVal.get().i64(); + uint32_t count = seg->bytes.length(); + + if (offset > memoryLength || memoryLength - offset < count) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_OUT_OF_BOUNDS); + return false; + } + memcpy(memoryBase + uintptr_t(offset), seg->bytes.begin(), count); + } + } + + return true; +} + +static const Import& FindImportFunction(const ImportVector& imports, + uint32_t funcImportIndex) { + for (const Import& import : imports) { + if (import.kind != DefinitionKind::Function) { + continue; + } + if (funcImportIndex == 0) { + return import; + } + funcImportIndex--; + } + MOZ_CRASH("ran out of imports"); +} + +bool Module::instantiateFunctions(JSContext* cx, + const JSObjectVector& funcImports) const { +#ifdef DEBUG + for (auto t : code().tiers()) { + MOZ_ASSERT(funcImports.length() == metadata(t).funcImports.length()); + } +#endif + + if (metadata().isAsmJS()) { + return true; + } + + Tier tier = code().stableTier(); + + for (size_t i = 0; i < metadata(tier).funcImports.length(); i++) { + if (!funcImports[i]->is()) { + continue; + } + + JSFunction* f = &funcImports[i]->as(); + if (!IsWasmExportedFunction(f)) { + continue; + } + + uint32_t funcIndex = ExportedFunctionToFuncIndex(f); + Instance& instance = ExportedFunctionToInstance(f); + Tier otherTier = instance.code().stableTier(); + + const FuncType& exportFuncType = instance.metadata().getFuncExportType( + instance.metadata(otherTier).lookupFuncExport(funcIndex)); + const FuncType& importFuncType = + metadata().getFuncImportType(metadata(tier).funcImports[i]); + + if (!FuncType::strictlyEquals(exportFuncType, importFuncType)) { + const Import& import = FindImportFunction(imports_, i); + UniqueChars importModuleName = import.module.toQuotedString(cx); + UniqueChars importFieldName = import.field.toQuotedString(cx); + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMPORT_SIG, + importModuleName.get(), importFieldName.get()); + return false; + } + } + + return true; +} + +template +static bool CheckLimits(JSContext* cx, T declaredMin, + const Maybe& declaredMax, T defaultMax, + T actualLength, const Maybe& actualMax, bool isAsmJS, + const char* kind) { + if (isAsmJS) { + MOZ_ASSERT(actualLength >= declaredMin); + MOZ_ASSERT(!declaredMax); + MOZ_ASSERT(actualLength == actualMax.value()); + return true; + } + + if (actualLength < declaredMin || + actualLength > declaredMax.valueOr(defaultMax)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMP_SIZE, kind); + return false; + } + + if ((actualMax && declaredMax && *actualMax > *declaredMax) || + (!actualMax && declaredMax)) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMP_MAX, kind); + return false; + } + + return true; +} + +static bool CheckSharing(JSContext* cx, bool declaredShared, bool isShared) { + if (isShared && + !cx->realm()->creationOptions().getSharedMemoryAndAtomicsEnabled()) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_NO_SHMEM_LINK); + return false; + } + + if (declaredShared && !isShared) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_IMP_SHARED_REQD); + return false; + } + + if (!declaredShared && isShared) { + JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr, + JSMSG_WASM_IMP_SHARED_BANNED); + return false; + } + + return true; +} + +// asm.js module instantiation supplies its own buffer, but for wasm, create and +// initialize the buffer if one is requested. Either way, the buffer is wrapped +// in a WebAssembly.Memory object which is what the Instance stores. +bool Module::instantiateMemory(JSContext* cx, + MutableHandle memory) const { + if (!metadata().usesMemory()) { + MOZ_ASSERT(!memory); + MOZ_ASSERT(AllSegmentsArePassive(dataSegments_)); + return true; + } + + MemoryDesc desc = *metadata().memory; + if (memory) { + MOZ_ASSERT_IF(metadata().isAsmJS(), memory->buffer().isPreparedForAsmJS()); + MOZ_ASSERT_IF(!metadata().isAsmJS(), memory->buffer().isWasm()); + + if (memory->indexType() != desc.indexType()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_IMP_INDEX, + ToString(memory->indexType())); + return false; + } + + if (!CheckLimits(cx, desc.initialPages(), desc.maximumPages(), + /* defaultMax */ MaxMemoryPages(desc.indexType()), + /* actualLength */ + memory->volatilePages(), memory->sourceMaxPages(), + metadata().isAsmJS(), "Memory")) { + return false; + } + + if (!CheckSharing(cx, desc.isShared(), memory->isShared())) { + return false; + } + } else { + MOZ_ASSERT(!metadata().isAsmJS()); + + if (desc.initialPages() > MaxMemoryPages(desc.indexType())) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_MEM_IMP_LIMIT); + return false; + } + + RootedArrayBufferObjectMaybeShared buffer(cx); + if (!CreateWasmBuffer(cx, desc, &buffer)) { + return false; + } + + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmMemory)); + memory.set(WasmMemoryObject::create( + cx, buffer, IsHugeMemoryEnabled(desc.indexType()), proto)); + if (!memory) { + return false; + } + } + + MOZ_RELEASE_ASSERT(memory->isHuge() == metadata().omitsBoundsChecks); + + return true; +} + +bool Module::instantiateTags(JSContext* cx, + WasmTagObjectVector& tagObjs) const { + size_t tagLength = metadata().tags.length(); + if (tagLength == 0) { + return true; + } + size_t importedTagsLength = tagObjs.length(); + if (tagObjs.length() <= tagLength && !tagObjs.resize(tagLength)) { + ReportOutOfMemory(cx); + return false; + } + + uint32_t tagIndex = 0; + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmTag)); + for (const TagDesc& desc : metadata().tags) { + if (tagIndex >= importedTagsLength) { + Rooted tagObj( + cx, WasmTagObject::create(cx, desc.type, proto)); + if (!tagObj) { + return false; + } + tagObjs[tagIndex] = tagObj; + } + tagIndex++; + } + return true; +} + +bool Module::instantiateImportedTable(JSContext* cx, const TableDesc& td, + Handle tableObj, + WasmTableObjectVector* tableObjs, + SharedTableVector* tables) const { + MOZ_ASSERT(tableObj); + MOZ_ASSERT(!metadata().isAsmJS()); + + Table& table = tableObj->table(); + if (!CheckLimits(cx, td.initialLength, td.maximumLength, + /* declaredMin */ MaxTableLimitField, + /* actualLength */ table.length(), table.maximum(), + metadata().isAsmJS(), "Table")) { + return false; + } + + if (!tables->append(&table)) { + ReportOutOfMemory(cx); + return false; + } + + if (!tableObjs->append(tableObj)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool Module::instantiateLocalTable(JSContext* cx, const TableDesc& td, + WasmTableObjectVector* tableObjs, + SharedTableVector* tables) const { + if (td.initialLength > MaxTableLength) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_TABLE_IMP_LIMIT); + return false; + } + + SharedTable table; + Rooted tableObj(cx); + if (td.isExported) { + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmTable)); + tableObj.set(WasmTableObject::create(cx, td.initialLength, td.maximumLength, + td.elemType, proto)); + if (!tableObj) { + return false; + } + table = &tableObj->table(); + } else { + table = Table::create(cx, td, /* Handle = */ nullptr); + if (!table) { + return false; + } + } + + // Note, appending a null pointer for non-exported local tables. + if (!tableObjs->append(tableObj.get())) { + ReportOutOfMemory(cx); + return false; + } + + if (!tables->emplaceBack(table)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool Module::instantiateTables(JSContext* cx, + const WasmTableObjectVector& tableImports, + MutableHandle tableObjs, + SharedTableVector* tables) const { + uint32_t tableIndex = 0; + for (const TableDesc& td : metadata().tables) { + if (tableIndex < tableImports.length()) { + Rooted tableObj(cx, tableImports[tableIndex]); + if (!instantiateImportedTable(cx, td, tableObj, &tableObjs.get(), + tables)) { + return false; + } + } else { + if (!instantiateLocalTable(cx, td, &tableObjs.get(), tables)) { + return false; + } + } + tableIndex++; + } + return true; +} + +static bool EnsureExportedGlobalObject(JSContext* cx, + const ValVector& globalImportValues, + size_t globalIndex, + const GlobalDesc& global, + WasmGlobalObjectVector& globalObjs) { + if (globalIndex < globalObjs.length() && globalObjs[globalIndex]) { + return true; + } + + RootedVal val(cx); + if (global.kind() == GlobalKind::Import) { + // If this is an import, then this must be a constant global that was + // provided without a global object. We must initialize it with the + // provided value while we still can differentiate this case. + MOZ_ASSERT(!global.isMutable()); + val.set(Val(globalImportValues[globalIndex])); + } else { + // If this is not an import, then the initial value will be set by + // Instance::init() for indirect globals or else by CreateExportObject(). + // In either case, we initialize with a default value here. + val.set(Val(global.type())); + } + + RootedObject proto(cx, &cx->global()->getPrototype(JSProto_WasmGlobal)); + Rooted go( + cx, WasmGlobalObject::create(cx, val, global.isMutable(), proto)); + if (!go) { + return false; + } + + if (globalObjs.length() <= globalIndex && + !globalObjs.resize(globalIndex + 1)) { + ReportOutOfMemory(cx); + return false; + } + + globalObjs[globalIndex] = go; + return true; +} + +bool Module::instantiateGlobals(JSContext* cx, + const ValVector& globalImportValues, + WasmGlobalObjectVector& globalObjs) const { + // If there are exported globals that aren't in globalObjs because they + // originate in this module or because they were immutable imports that came + // in as primitive values then we must create cells in the globalObjs for + // them here, as WasmInstanceObject::create() and CreateExportObject() will + // need the cells to exist. + + const GlobalDescVector& globals = metadata().globals; + + for (const Export& exp : exports_) { + if (exp.kind() != DefinitionKind::Global) { + continue; + } + unsigned globalIndex = exp.globalIndex(); + const GlobalDesc& global = globals[globalIndex]; + if (!EnsureExportedGlobalObject(cx, globalImportValues, globalIndex, global, + globalObjs)) { + return false; + } + } + + // Imported globals that are not re-exported may also have received only a + // primitive value; these globals are always immutable. Assert that we do + // not need to create any additional Global objects for such imports. + +#ifdef DEBUG + size_t numGlobalImports = 0; + for (const Import& import : imports_) { + if (import.kind != DefinitionKind::Global) { + continue; + } + size_t globalIndex = numGlobalImports++; + const GlobalDesc& global = globals[globalIndex]; + MOZ_ASSERT(global.importIndex() == globalIndex); + MOZ_ASSERT_IF(global.isIndirect(), + globalIndex < globalObjs.length() || globalObjs[globalIndex]); + } + MOZ_ASSERT_IF(!metadata().isAsmJS(), + numGlobalImports == globals.length() || + !globals[numGlobalImports].isImport()); +#endif + return true; +} + +static bool GetFunctionExport(JSContext* cx, + Handle instanceObj, + const JSObjectVector& funcImports, + uint32_t funcIndex, MutableHandleFunction func) { + if (funcIndex < funcImports.length() && + funcImports[funcIndex]->is()) { + JSFunction* f = &funcImports[funcIndex]->as(); + if (IsWasmExportedFunction(f)) { + func.set(f); + return true; + } + } + + return instanceObj->getExportedFunction(cx, instanceObj, funcIndex, func); +} + +static bool GetGlobalExport(JSContext* cx, + Handle instanceObj, + const JSObjectVector& funcImports, + const GlobalDesc& global, uint32_t globalIndex, + const ValVector& globalImportValues, + const WasmGlobalObjectVector& globalObjs, + MutableHandleValue val) { + // A global object for this index is guaranteed to exist by + // instantiateGlobals. + Rooted globalObj(cx, globalObjs[globalIndex]); + val.setObject(*globalObj); + + // We are responsible to set the initial value of the global object here if + // it's not imported or indirect. Imported global objects have their initial + // value set by their defining module, or are set by + // EnsureExportedGlobalObject when a constant value is provided as an import. + // Indirect exported globals that are not imported, are initialized in + // Instance::init. + if (global.isIndirect() || global.isImport()) { + return true; + } + + // This must be an exported immutable global defined in this module. The + // instance either has compiled the value into the code or has its own copy + // in its global data area. Either way, we must initialize the global object + // with the same initial value. + MOZ_ASSERT(!global.isMutable()); + MOZ_ASSERT(!global.isImport()); + RootedVal globalVal(cx); + MOZ_RELEASE_ASSERT(!global.isImport()); + const InitExpr& init = global.initExpr(); + if (!init.evaluate(cx, instanceObj, &globalVal)) { + return false; + } + globalObj->val() = globalVal; + return true; +} + +static bool CreateExportObject( + JSContext* cx, Handle instanceObj, + const JSObjectVector& funcImports, const WasmTableObjectVector& tableObjs, + Handle memoryObj, const WasmTagObjectVector& tagObjs, + const ValVector& globalImportValues, + const WasmGlobalObjectVector& globalObjs, const ExportVector& exports) { + const Instance& instance = instanceObj->instance(); + const Metadata& metadata = instance.metadata(); + const GlobalDescVector& globals = metadata.globals; + + if (metadata.isAsmJS() && exports.length() == 1 && + exports[0].fieldName().isEmpty()) { + RootedFunction func(cx); + if (!GetFunctionExport(cx, instanceObj, funcImports, exports[0].funcIndex(), + &func)) { + return false; + } + instanceObj->initExportsObj(*func.get()); + return true; + } + + RootedObject exportObj(cx); + uint8_t propertyAttr = JSPROP_ENUMERATE; + + if (metadata.isAsmJS()) { + exportObj = NewPlainObject(cx); + } else { + exportObj = NewPlainObjectWithProto(cx, nullptr); + propertyAttr |= JSPROP_READONLY | JSPROP_PERMANENT; + } + if (!exportObj) { + return false; + } + + for (const Export& exp : exports) { + JSAtom* atom = exp.fieldName().toAtom(cx); + if (!atom) { + return false; + } + + RootedId id(cx, AtomToId(atom)); + RootedValue val(cx); + switch (exp.kind()) { + case DefinitionKind::Function: { + RootedFunction func(cx); + if (!GetFunctionExport(cx, instanceObj, funcImports, exp.funcIndex(), + &func)) { + return false; + } + val = ObjectValue(*func); + break; + } + case DefinitionKind::Table: { + val = ObjectValue(*tableObjs[exp.tableIndex()]); + break; + } + case DefinitionKind::Memory: { + val = ObjectValue(*memoryObj); + break; + } + case DefinitionKind::Global: { + const GlobalDesc& global = globals[exp.globalIndex()]; + if (!GetGlobalExport(cx, instanceObj, funcImports, global, + exp.globalIndex(), globalImportValues, globalObjs, + &val)) { + return false; + } + break; + } + case DefinitionKind::Tag: { + val = ObjectValue(*tagObjs[exp.tagIndex()]); + break; + } + } + + if (!JS_DefinePropertyById(cx, exportObj, id, val, propertyAttr)) { + return false; + } + } + + if (!metadata.isAsmJS()) { + if (!PreventExtensions(cx, exportObj)) { + return false; + } + } + + instanceObj->initExportsObj(*exportObj); + return true; +} + +bool Module::instantiate(JSContext* cx, ImportValues& imports, + HandleObject instanceProto, + MutableHandle instance) const { + MOZ_RELEASE_ASSERT(cx->wasm().haveSignalHandlers); + + if (!instantiateFunctions(cx, imports.funcs)) { + return false; + } + + Rooted memory(cx, imports.memory); + if (!instantiateMemory(cx, &memory)) { + return false; + } + + // Note that the following will extend imports.exceptionObjs with wrappers for + // the local (non-imported) exceptions of the module. + // The resulting vector is sparse, i.e., it will be null in slots that contain + // exceptions that are neither exported or imported. + // On the contrary, all the slots of exceptionTags will be filled with + // unique tags. + + if (!instantiateTags(cx, imports.tagObjs)) { + return false; + } + + // Note that tableObjs is sparse: it will be null in slots that contain + // tables that are neither exported nor imported. + + Rooted tableObjs(cx); + SharedTableVector tables; + if (!instantiateTables(cx, imports.tables, &tableObjs, &tables)) { + return false; + } + + if (!instantiateGlobals(cx, imports.globalValues, imports.globalObjs)) { + return false; + } + + UniqueDebugState maybeDebug; + if (metadata().debugEnabled) { + maybeDebug = cx->make_unique(*code_, *this); + if (!maybeDebug) { + ReportOutOfMemory(cx); + return false; + } + } + + instance.set(WasmInstanceObject::create( + cx, code_, dataSegments_, elemSegments_, metadata().instanceDataLength, + memory, std::move(tables), imports.funcs, metadata().globals, + imports.globalValues, imports.globalObjs, imports.tagObjs, instanceProto, + std::move(maybeDebug))); + if (!instance) { + return false; + } + + if (!CreateExportObject(cx, instance, imports.funcs, tableObjs.get(), memory, + imports.tagObjs, imports.globalValues, + imports.globalObjs, exports_)) { + return false; + } + + // Register the instance with the Realm so that it can find out about global + // events like profiling being enabled in the realm. Registration does not + // require a fully-initialized instance and must precede initSegments as the + // final pre-requisite for a live instance. + + if (!cx->realm()->wasm.registerInstance(cx, instance)) { + ReportOutOfMemory(cx); + return false; + } + + // Perform initialization as the final step after the instance is fully + // constructed since this can make the instance live to content (even if the + // start function fails). + + if (!initSegments(cx, instance, memory)) { + return false; + } + + // Now that the instance is fully live and initialized, the start function. + // Note that failure may cause instantiation to throw, but the instance may + // still be live via edges created by initSegments or the start function. + + if (metadata().startFuncIndex) { + FixedInvokeArgs<0> args(cx); + if (!instance->instance().callExport(cx, *metadata().startFuncIndex, + args)) { + return false; + } + } + + JSUseCounter useCounter = + metadata().isAsmJS() ? JSUseCounter::ASMJS : JSUseCounter::WASM; + cx->runtime()->setUseCounter(instance, useCounter); + + if (cx->options().testWasmAwaitTier2()) { + testingBlockOnTier2Complete(); + } + + return true; +} diff --git a/js/src/wasm/WasmModule.h b/js/src/wasm/WasmModule.h new file mode 100644 index 0000000000..e070f749c9 --- /dev/null +++ b/js/src/wasm/WasmModule.h @@ -0,0 +1,225 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_module_h +#define wasm_module_h + +#include "js/WasmModule.h" +#include "js/BuildId.h" + +#include "wasm/WasmCode.h" +#include "wasm/WasmException.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmTable.h" + +namespace JS { +class OptimizedEncodingListener; +} + +namespace js { +namespace wasm { + +struct CompileArgs; + +// In the context of wasm, the OptimizedEncodingListener specifically is +// listening for the completion of tier-2. + +using Tier2Listener = RefPtr; + +// A struct containing the typed, imported values that are harvested from the +// import object and passed to Module::instantiate(). This struct must be +// stored in a (Persistent)Rooted, not in the heap due to its use of TraceRoot() +// and complete lack of barriers. + +struct ImportValues { + JSObjectVector funcs; + WasmTableObjectVector tables; + WasmMemoryObject* memory; + WasmTagObjectVector tagObjs; + WasmGlobalObjectVector globalObjs; + ValVector globalValues; + + ImportValues() : memory(nullptr) {} + + void trace(JSTracer* trc) { + funcs.trace(trc); + tables.trace(trc); + if (memory) { + TraceRoot(trc, &memory, "import values memory"); + } + tagObjs.trace(trc); + globalObjs.trace(trc); + globalValues.trace(trc); + } +}; + +// Module represents a compiled wasm module and primarily provides three +// operations: instantiation, tiered compilation, serialization. A Module can be +// instantiated any number of times to produce new Instance objects. A Module +// can have a single tier-2 task initiated to augment a Module's code with a +// higher tier. A Module can have its optimized code serialized at any point +// where the LinkData is also available, which is primarily (1) at the end of +// module generation, (2) at the end of tier-2 compilation. +// +// Fully linked-and-instantiated code (represented by SharedCode and its owned +// ModuleSegment) can be shared between instances. + +class Module : public JS::WasmModule { + const SharedCode code_; + const ImportVector imports_; + const ExportVector exports_; + const DataSegmentVector dataSegments_; + const ElemSegmentVector elemSegments_; + const CustomSectionVector customSections_; + + // This field is only meaningful when code_->metadata().debugEnabled. + + const SharedBytes debugBytecode_; + + // This field is set during tier-2 compilation and cleared on success or + // failure. These happen on different threads and are serialized by the + // control flow of helper tasks. + + mutable Tier2Listener tier2Listener_; + + // This flag is used for logging (and testing) purposes to indicate + // whether the module was deserialized (from a cache). + + const bool loggingDeserialized_; + + // This flag is only used for testing purposes and is cleared on success or + // failure. The field is racily polled from various threads. + + mutable Atomic testingTier2Active_; + + // Cached malloc allocation size for GC memory tracking. + + size_t gcMallocBytesExcludingCode_; + + bool instantiateFunctions(JSContext* cx, + const JSObjectVector& funcImports) const; + bool instantiateMemory(JSContext* cx, + MutableHandle memory) const; + bool instantiateTags(JSContext* cx, WasmTagObjectVector& tagObjs) const; + bool instantiateImportedTable(JSContext* cx, const TableDesc& td, + Handle table, + WasmTableObjectVector* tableObjs, + SharedTableVector* tables) const; + bool instantiateLocalTable(JSContext* cx, const TableDesc& td, + WasmTableObjectVector* tableObjs, + SharedTableVector* tables) const; + bool instantiateTables(JSContext* cx, + const WasmTableObjectVector& tableImports, + MutableHandle tableObjs, + SharedTableVector* tables) const; + bool instantiateGlobals(JSContext* cx, const ValVector& globalImportValues, + WasmGlobalObjectVector& globalObjs) const; + bool initSegments(JSContext* cx, Handle instance, + Handle memory) const; + + class Tier2GeneratorTaskImpl; + + public: + Module(const Code& code, ImportVector&& imports, ExportVector&& exports, + DataSegmentVector&& dataSegments, ElemSegmentVector&& elemSegments, + CustomSectionVector&& customSections, + const ShareableBytes* debugBytecode = nullptr, + bool loggingDeserialized = false) + : code_(&code), + imports_(std::move(imports)), + exports_(std::move(exports)), + dataSegments_(std::move(dataSegments)), + elemSegments_(std::move(elemSegments)), + customSections_(std::move(customSections)), + debugBytecode_(debugBytecode), + loggingDeserialized_(loggingDeserialized), + testingTier2Active_(false) { + initGCMallocBytesExcludingCode(); + } + ~Module() override; + + const Code& code() const { return *code_; } + const ModuleSegment& moduleSegment(Tier t) const { return code_->segment(t); } + const Metadata& metadata() const { return code_->metadata(); } + const MetadataTier& metadata(Tier t) const { return code_->metadata(t); } + const ImportVector& imports() const { return imports_; } + const ExportVector& exports() const { return exports_; } + const CustomSectionVector& customSections() const { return customSections_; } + const Bytes& debugBytecode() const { return debugBytecode_->bytes; } + uint32_t codeLength(Tier t) const { return code_->segment(t).length(); } + + // Instantiate this module with the given imports: + + bool instantiate(JSContext* cx, ImportValues& imports, + HandleObject instanceProto, + MutableHandle instanceObj) const; + + // Tier-2 compilation may be initiated after the Module is constructed at + // most once. When tier-2 compilation completes, ModuleGenerator calls + // finishTier2() from a helper thread, passing tier-variant data which will + // be installed and made visible. + + void startTier2(const CompileArgs& args, const ShareableBytes& bytecode, + JS::OptimizedEncodingListener* listener); + bool finishTier2(const LinkData& linkData2, UniqueCodeTier code2) const; + + void testingBlockOnTier2Complete() const; + bool testingTier2Active() const { return testingTier2Active_; } + + // Code caching support. + + [[nodiscard]] bool serialize(const LinkData& linkData, Bytes* bytes) const; + static RefPtr deserialize(const uint8_t* begin, size_t size); + bool loggingDeserialized() const { return loggingDeserialized_; } + + // JS API and JS::WasmModule implementation: + + JSObject* createObject(JSContext* cx) const override; + JSObject* createObjectForAsmJS(JSContext* cx) const override; + + // about:memory reporting: + + void addSizeOfMisc(MallocSizeOf mallocSizeOf, Metadata::SeenSet* seenMetadata, + Code::SeenSet* seenCode, size_t* code, size_t* data) const; + + // GC malloc memory tracking: + + void initGCMallocBytesExcludingCode(); + size_t gcMallocBytesExcludingCode() const { + return gcMallocBytesExcludingCode_; + } + + // Generated code analysis support: + + bool extractCode(JSContext* cx, Tier tier, MutableHandleValue vp) const; + + WASM_DECLARE_FRIEND_SERIALIZE_ARGS(Module, const wasm::LinkData& linkData); +}; + +using MutableModule = RefPtr; +using SharedModule = RefPtr; + +// JS API implementations: + +[[nodiscard]] bool GetOptimizedEncodingBuildId(JS::BuildIdCharVector* buildId); + +} // namespace wasm +} // namespace js + +#endif // wasm_module_h diff --git a/js/src/wasm/WasmModuleTypes.cpp b/js/src/wasm/WasmModuleTypes.cpp new file mode 100644 index 0000000000..9823d7ddbc --- /dev/null +++ b/js/src/wasm/WasmModuleTypes.cpp @@ -0,0 +1,171 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmModuleTypes.h" + +#include "mozilla/Range.h" + +#include "vm/JSAtom.h" +#include "vm/MallocProvider.h" +#include "wasm/WasmUtility.h" + +#include "vm/JSAtom-inl.h" + +using namespace js; +using namespace js::wasm; + +/* static */ +CacheableName CacheableName::fromUTF8Chars(UniqueChars&& utf8Chars) { + size_t length = strlen(utf8Chars.get()); + UTF8Bytes bytes; + bytes.replaceRawBuffer(utf8Chars.release(), length, length + 1); + return CacheableName(std::move(bytes)); +} + +/* static */ +bool CacheableName::fromUTF8Chars(const char* utf8Chars, CacheableName* name) { + size_t utf8CharsLen = strlen(utf8Chars); + UTF8Bytes bytes; + if (!bytes.resizeUninitialized(utf8CharsLen)) { + return false; + } + memcpy(bytes.begin(), utf8Chars, utf8CharsLen); + *name = CacheableName(std::move(bytes)); + return true; +} + +JSAtom* CacheableName::toAtom(JSContext* cx) const { + return AtomizeUTF8Chars(cx, begin(), length()); +} + +bool CacheableName::toPropertyKey(JSContext* cx, + MutableHandleId propertyKey) const { + JSAtom* atom = toAtom(cx); + if (!atom) { + return false; + } + propertyKey.set(AtomToId(atom)); + return true; +} + +UniqueChars CacheableName::toQuotedString(JSContext* cx) const { + RootedString atom(cx, toAtom(cx)); + if (!atom) { + return nullptr; + } + return QuoteString(cx, atom.get()); +} + +size_t CacheableName::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return bytes_.sizeOfExcludingThis(mallocSizeOf); +} + +size_t Import::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return module.sizeOfExcludingThis(mallocSizeOf) + + field.sizeOfExcludingThis(mallocSizeOf); +} + +Export::Export(CacheableName&& fieldName, uint32_t index, DefinitionKind kind) + : fieldName_(std::move(fieldName)) { + pod.kind_ = kind; + pod.index_ = index; +} + +Export::Export(CacheableName&& fieldName, DefinitionKind kind) + : fieldName_(std::move(fieldName)) { + pod.kind_ = kind; + pod.index_ = 0; +} + +uint32_t Export::funcIndex() const { + MOZ_ASSERT(pod.kind_ == DefinitionKind::Function); + return pod.index_; +} + +uint32_t Export::globalIndex() const { + MOZ_ASSERT(pod.kind_ == DefinitionKind::Global); + return pod.index_; +} + +uint32_t Export::tagIndex() const { + MOZ_ASSERT(pod.kind_ == DefinitionKind::Tag); + return pod.index_; +} + +uint32_t Export::tableIndex() const { + MOZ_ASSERT(pod.kind_ == DefinitionKind::Table); + return pod.index_; +} + +size_t Export::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return fieldName_.sizeOfExcludingThis(mallocSizeOf); +} + +size_t GlobalDesc::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return initial_.sizeOfExcludingThis(mallocSizeOf); +} + +bool TagType::initialize(ValTypeVector&& argTypes) { + MOZ_ASSERT(argTypes_.empty() && argOffsets_.empty() && size_ == 0); + + argTypes_ = std::move(argTypes); + if (!argOffsets_.resize(argTypes_.length())) { + return false; + } + + StructLayout layout; + for (size_t i = 0; i < argTypes_.length(); i++) { + CheckedInt32 offset = layout.addField(FieldType(argTypes_[i].packed())); + if (!offset.isValid()) { + return false; + } + argOffsets_[i] = offset.value(); + } + + CheckedInt32 size = layout.close(); + if (!size.isValid()) { + return false; + } + this->size_ = size.value(); + + return true; +} + +size_t TagType::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return argTypes_.sizeOfExcludingThis(mallocSizeOf) + + argOffsets_.sizeOfExcludingThis(mallocSizeOf); +} + +size_t TagDesc::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return type->sizeOfExcludingThis(mallocSizeOf); +} + +size_t ElemSegment::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return SizeOfMaybeExcludingThis(offsetIfActive, mallocSizeOf) + + elemFuncIndices.sizeOfExcludingThis(mallocSizeOf); +} + +size_t DataSegment::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return SizeOfMaybeExcludingThis(offsetIfActive, mallocSizeOf) + + bytes.sizeOfExcludingThis(mallocSizeOf); +} + +size_t CustomSection::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return name.sizeOfExcludingThis(mallocSizeOf) + sizeof(*payload) + + payload->sizeOfExcludingThis(mallocSizeOf); +} diff --git a/js/src/wasm/WasmModuleTypes.h b/js/src/wasm/WasmModuleTypes.h new file mode 100644 index 0000000000..9d018bdc01 --- /dev/null +++ b/js/src/wasm/WasmModuleTypes.h @@ -0,0 +1,632 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_module_types_h +#define wasm_module_types_h + +#include "mozilla/RefPtr.h" +#include "mozilla/Span.h" + +#include "js/AllocPolicy.h" +#include "js/RefCounted.h" +#include "js/Utility.h" +#include "js/Vector.h" + +#include "wasm/WasmCompileArgs.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmExprType.h" +#include "wasm/WasmInitExpr.h" +#include "wasm/WasmMemory.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmShareable.h" +#include "wasm/WasmTypeDecls.h" +#include "wasm/WasmValType.h" +#include "wasm/WasmValue.h" + +namespace js { +namespace wasm { + +using mozilla::Maybe; +using mozilla::Nothing; +using mozilla::Span; + +class FuncType; + +// A Module can either be asm.js or wasm. + +enum ModuleKind { Wasm, AsmJS }; + +// CacheableChars is used to cacheably store UniqueChars. + +struct CacheableChars : UniqueChars { + CacheableChars() = default; + explicit CacheableChars(char* ptr) : UniqueChars(ptr) {} + MOZ_IMPLICIT CacheableChars(UniqueChars&& rhs) + : UniqueChars(std::move(rhs)) {} + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using CacheableCharsVector = Vector; + +// CacheableName is used to cacheably store a UTF-8 string that may contain +// null terminators in sequence. + +struct CacheableName { + private: + UTF8Bytes bytes_; + + const char* begin() const { return (const char*)bytes_.begin(); } + size_t length() const { return bytes_.length(); } + + public: + CacheableName() = default; + MOZ_IMPLICIT CacheableName(UTF8Bytes&& rhs) : bytes_(std::move(rhs)) {} + + bool isEmpty() const { return bytes_.length() == 0; } + + Span utf8Bytes() { return Span(bytes_); } + Span utf8Bytes() const { return Span(bytes_); } + + static CacheableName fromUTF8Chars(UniqueChars&& utf8Chars); + [[nodiscard]] static bool fromUTF8Chars(const char* utf8Chars, + CacheableName* name); + + [[nodiscard]] JSAtom* toAtom(JSContext* cx) const; + [[nodiscard]] bool toPropertyKey(JSContext* cx, + MutableHandleId propertyKey) const; + [[nodiscard]] UniqueChars toQuotedString(JSContext* cx) const; + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(CacheableName); +}; + +using CacheableNameVector = Vector; + +// A hash policy for names. +struct NameHasher { + using Key = Span; + using Lookup = Span; + + static HashNumber hash(const Lookup& aLookup) { + return mozilla::HashString(aLookup.data(), aLookup.Length()); + } + + static bool match(const Key& aKey, const Lookup& aLookup) { + return aKey == aLookup; + } +}; + +// Import describes a single wasm import. An ImportVector describes all +// of a single module's imports. +// +// ImportVector is built incrementally by ModuleGenerator and then stored +// immutably by Module. + +struct Import { + CacheableName module; + CacheableName field; + DefinitionKind kind; + + Import() = default; + Import(CacheableName&& module, CacheableName&& field, DefinitionKind kind) + : module(std::move(module)), field(std::move(field)), kind(kind) {} + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using ImportVector = Vector; + +// Export describes the export of a definition in a Module to a field in the +// export object. The Export stores the index of the exported item in the +// appropriate type-specific module data structure (function table, global +// table, table table, and - eventually - memory table). +// +// Note a single definition can be exported by multiple Exports in the +// ExportVector. +// +// ExportVector is built incrementally by ModuleGenerator and then stored +// immutably by Module. + +class Export { + public: + struct CacheablePod { + DefinitionKind kind_; + uint32_t index_; + + WASM_CHECK_CACHEABLE_POD(kind_, index_); + }; + + private: + CacheableName fieldName_; + CacheablePod pod; + + public: + Export() = default; + explicit Export(CacheableName&& fieldName, uint32_t index, + DefinitionKind kind); + explicit Export(CacheableName&& fieldName, DefinitionKind kind); + + const CacheableName& fieldName() const { return fieldName_; } + + DefinitionKind kind() const { return pod.kind_; } + uint32_t funcIndex() const; + uint32_t tagIndex() const; + uint32_t globalIndex() const; + uint32_t tableIndex() const; + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(Export); +}; + +WASM_DECLARE_CACHEABLE_POD(Export::CacheablePod); + +using ExportVector = Vector; + +// FuncFlags provides metadata for a function definition. + +enum class FuncFlags : uint8_t { + None = 0x0, + // The function maybe be accessible by JS and needs thunks generated for it. + // See `[SMDOC] Exported wasm functions and the jit-entry stubs` in + // WasmJS.cpp for more information. + Exported = 0x1, + // The function should have thunks generated upon instantiation, not upon + // first call. May only be set if `Exported` is set. + Eager = 0x2, + // The function can be the target of a ref.func instruction in the code + // section. May only be set if `Exported` is set. + CanRefFunc = 0x4, +}; + +// A FuncDesc describes a single function definition. + +struct FuncDesc { + const FuncType* type; + // Bit pack to keep this struct small on 32-bit systems + uint32_t typeIndex : 24; + FuncFlags flags : 8; + + // Assert that the bit packing scheme is viable + static_assert(MaxTypes <= (1 << 24) - 1); + static_assert(sizeof(FuncFlags) == sizeof(uint8_t)); + + FuncDesc() = default; + FuncDesc(const FuncType* type, uint32_t typeIndex) + : type(type), typeIndex(typeIndex), flags(FuncFlags::None) {} + + bool isExported() const { + return uint8_t(flags) & uint8_t(FuncFlags::Exported); + } + bool isEager() const { return uint8_t(flags) & uint8_t(FuncFlags::Eager); } + bool canRefFunc() const { + return uint8_t(flags) & uint8_t(FuncFlags::CanRefFunc); + } +}; + +using FuncDescVector = Vector; + +// A GlobalDesc describes a single global variable. +// +// wasm can import and export mutable and immutable globals. +// +// asm.js can import mutable and immutable globals, but a mutable global has a +// location that is private to the module, and its initial value is copied into +// that cell from the environment. asm.js cannot export globals. + +enum class GlobalKind { Import, Constant, Variable }; + +class GlobalDesc { + GlobalKind kind_; + // Stores the value type of this global for all kinds, and the initializer + // expression when `constant` or `variable`. + InitExpr initial_; + // Metadata for the global when `variable` or `import`. + unsigned offset_; + bool isMutable_; + bool isWasm_; + bool isExport_; + // Metadata for the global when `import`. + uint32_t importIndex_; + + // Private, as they have unusual semantics. + + bool isExport() const { return !isConstant() && isExport_; } + bool isWasm() const { return !isConstant() && isWasm_; } + + public: + GlobalDesc() = default; + + explicit GlobalDesc(InitExpr&& initial, bool isMutable, + ModuleKind kind = ModuleKind::Wasm) + : kind_((isMutable || !initial.isLiteral()) ? GlobalKind::Variable + : GlobalKind::Constant) { + initial_ = std::move(initial); + if (isVariable()) { + isMutable_ = isMutable; + isWasm_ = kind == Wasm; + isExport_ = false; + offset_ = UINT32_MAX; + } + } + + explicit GlobalDesc(ValType type, bool isMutable, uint32_t importIndex, + ModuleKind kind = ModuleKind::Wasm) + : kind_(GlobalKind::Import) { + initial_ = InitExpr(LitVal(type)); + importIndex_ = importIndex; + isMutable_ = isMutable; + isWasm_ = kind == Wasm; + isExport_ = false; + offset_ = UINT32_MAX; + } + + void setOffset(unsigned offset) { + MOZ_ASSERT(!isConstant()); + MOZ_ASSERT(offset_ == UINT32_MAX); + offset_ = offset; + } + unsigned offset() const { + MOZ_ASSERT(!isConstant()); + MOZ_ASSERT(offset_ != UINT32_MAX); + return offset_; + } + + void setIsExport() { + if (!isConstant()) { + isExport_ = true; + } + } + + GlobalKind kind() const { return kind_; } + bool isVariable() const { return kind_ == GlobalKind::Variable; } + bool isConstant() const { return kind_ == GlobalKind::Constant; } + bool isImport() const { return kind_ == GlobalKind::Import; } + + bool isMutable() const { return !isConstant() && isMutable_; } + const InitExpr& initExpr() const { + MOZ_ASSERT(!isImport()); + return initial_; + } + uint32_t importIndex() const { + MOZ_ASSERT(isImport()); + return importIndex_; + } + + LitVal constantValue() const { return initial_.literal(); } + + // If isIndirect() is true then storage for the value is not in the + // instance's global area, but in a WasmGlobalObject::Cell hanging off a + // WasmGlobalObject; the global area contains a pointer to the Cell. + // + // We don't want to indirect unless we must, so only mutable, exposed + // globals are indirected - in all other cases we copy values into and out + // of their module. + // + // Note that isIndirect() isn't equivalent to getting a WasmGlobalObject: + // an immutable exported global will still get an object, but will not be + // indirect. + bool isIndirect() const { + return isMutable() && isWasm() && (isImport() || isExport()); + } + + ValType type() const { return initial_.type(); } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(GlobalDesc); +}; + +using GlobalDescVector = Vector; + +// A TagDesc represents fresh per-instance tags that are used for the +// exception handling proposal and potentially other future proposals. + +// The TagOffsetVector represents the offsets in the layout of the +// data buffer stored in a Wasm exception. +using TagOffsetVector = Vector; + +struct TagType : AtomicRefCounted { + ValTypeVector argTypes_; + TagOffsetVector argOffsets_; + uint32_t size_; + + TagType() : size_(0) {} + + ResultType resultType() const { return ResultType::Vector(argTypes_); } + + [[nodiscard]] bool initialize(ValTypeVector&& argTypes); + + [[nodiscard]] bool clone(const TagType& src) { + MOZ_ASSERT(argTypes_.empty() && argOffsets_.empty() && size_ == 0); + if (!argTypes_.appendAll(src.argTypes_) || + !argOffsets_.appendAll(src.argOffsets_)) { + return false; + } + size_ = src.size_; + return true; + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using MutableTagType = RefPtr; +using SharedTagType = RefPtr; + +struct TagDesc { + TagKind kind; + SharedTagType type; + bool isExport; + + TagDesc() : isExport(false) {} + TagDesc(TagKind kind, const SharedTagType& type, bool isExport = false) + : kind(kind), type(type), isExport(isExport) {} + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using TagDescVector = Vector; + +// When a ElemSegment is "passive" it is shared between a wasm::Module and its +// wasm::Instances. To allow each segment to be released as soon as the last +// Instance elem.drops it and the Module is destroyed, each ElemSegment is +// individually atomically ref-counted. + +struct ElemSegment : AtomicRefCounted { + enum class Kind { + Active, + Passive, + Declared, + }; + + Kind kind; + uint32_t tableIndex; + RefType elemType; + Maybe offsetIfActive; + Uint32Vector elemFuncIndices; // Element may be NullFuncIndex + + bool active() const { return kind == Kind::Active; } + + const InitExpr& offset() const { return *offsetIfActive; } + + size_t length() const { return elemFuncIndices.length(); } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +// NullFuncIndex represents the case when an element segment (of type funcref) +// contains a null element. +constexpr uint32_t NullFuncIndex = UINT32_MAX; +static_assert(NullFuncIndex > MaxFuncs, "Invariant"); + +using MutableElemSegment = RefPtr; +using SharedElemSegment = RefPtr; +using ElemSegmentVector = Vector; + +// DataSegmentEnv holds the initial results of decoding a data segment from the +// bytecode and is stored in the ModuleEnvironment during compilation. When +// compilation completes, (non-Env) DataSegments are created and stored in +// the wasm::Module which contain copies of the data segment payload. This +// allows non-compilation uses of wasm validation to avoid expensive copies. +// +// When a DataSegment is "passive" it is shared between a wasm::Module and its +// wasm::Instances. To allow each segment to be released as soon as the last +// Instance mem.drops it and the Module is destroyed, each DataSegment is +// individually atomically ref-counted. + +struct DataSegmentEnv { + Maybe offsetIfActive; + uint32_t bytecodeOffset; + uint32_t length; +}; + +using DataSegmentEnvVector = Vector; + +struct DataSegment : AtomicRefCounted { + Maybe offsetIfActive; + Bytes bytes; + + DataSegment() = default; + + bool active() const { return !!offsetIfActive; } + + const InitExpr& offset() const { return *offsetIfActive; } + + [[nodiscard]] bool init(const ShareableBytes& bytecode, + const DataSegmentEnv& src) { + if (src.offsetIfActive) { + offsetIfActive.emplace(); + if (!offsetIfActive->clone(*src.offsetIfActive)) { + return false; + } + } + return bytes.append(bytecode.begin() + src.bytecodeOffset, src.length); + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using MutableDataSegment = RefPtr; +using SharedDataSegment = RefPtr; +using DataSegmentVector = Vector; + +// The CustomSection(Env) structs are like DataSegment(Env): CustomSectionEnv is +// stored in the ModuleEnvironment and CustomSection holds a copy of the payload +// and is stored in the wasm::Module. + +struct CustomSectionEnv { + uint32_t nameOffset; + uint32_t nameLength; + uint32_t payloadOffset; + uint32_t payloadLength; +}; + +using CustomSectionEnvVector = Vector; + +struct CustomSection { + Bytes name; + SharedBytes payload; + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +using CustomSectionVector = Vector; + +// A Name represents a string of utf8 chars embedded within the name custom +// section. The offset of a name is expressed relative to the beginning of the +// name section's payload so that Names can stored in wasm::Code, which only +// holds the name section's bytes, not the whole bytecode. + +struct Name { + // All fields are treated as cacheable POD: + uint32_t offsetInNamePayload; + uint32_t length; + + WASM_CHECK_CACHEABLE_POD(offsetInNamePayload, length); + + Name() : offsetInNamePayload(UINT32_MAX), length(0) {} +}; + +WASM_DECLARE_CACHEABLE_POD(Name); + +using NameVector = Vector; + +// The kind of limits to decode or convert from JS. + +enum class LimitsKind { + Memory, + Table, +}; + +// Represents the resizable limits of memories and tables. + +struct Limits { + // `indexType` will always be I32 for tables, but may be I64 for memories + // when memory64 is enabled. + IndexType indexType; + + // The initial and maximum limit. The unit is pages for memories and elements + // for tables. + uint64_t initial; + Maybe maximum; + + // `shared` is Shareable::False for tables but may be Shareable::True for + // memories. + Shareable shared; + + WASM_CHECK_CACHEABLE_POD(indexType, initial, maximum, shared); + + Limits() = default; + explicit Limits(uint64_t initial, const Maybe& maximum = Nothing(), + Shareable shared = Shareable::False) + : indexType(IndexType::I32), + initial(initial), + maximum(maximum), + shared(shared) {} +}; + +WASM_DECLARE_CACHEABLE_POD(Limits); + +// MemoryDesc describes a memory. + +struct MemoryDesc { + Limits limits; + + WASM_CHECK_CACHEABLE_POD(limits); + + bool isShared() const { return limits.shared == Shareable::True; } + + // Whether a backing store for this memory may move when grown. + bool canMovingGrow() const { return limits.maximum.isNothing(); } + + // Whether the bounds check limit (see the doc comment in + // ArrayBufferObject.cpp regarding linear memory structure) can ever be + // larger than 32-bits. + bool boundsCheckLimitIs32Bits() const { + return limits.maximum.isSome() && + limits.maximum.value() < (0x100000000 / PageSize); + } + + IndexType indexType() const { return limits.indexType; } + + // The initial length of this memory in pages. + Pages initialPages() const { return Pages(limits.initial); } + + // The maximum length of this memory in pages. + Maybe maximumPages() const { + return limits.maximum.map([](uint64_t x) { return Pages(x); }); + } + + // The initial length of this memory in bytes. Only valid for memory32. + uint64_t initialLength32() const { + MOZ_ASSERT(indexType() == IndexType::I32); + // See static_assert after MemoryDesc for why this is safe. + return limits.initial * PageSize; + } + + uint64_t initialLength64() const { + MOZ_ASSERT(indexType() == IndexType::I64); + return limits.initial * PageSize; + } + + MemoryDesc() = default; + explicit MemoryDesc(Limits limits) : limits(limits) {} +}; + +WASM_DECLARE_CACHEABLE_POD(MemoryDesc); + +// We don't need to worry about overflow with a Memory32 field when +// using a uint64_t. +static_assert(MaxMemory32LimitField <= UINT64_MAX / PageSize); + +// TableDesc describes a table as well as the offset of the table's base pointer +// in global memory. +// +// A TableDesc contains the element type and whether the table is for asm.js, +// which determines the table representation. +// - ExternRef: a wasm anyref word (wasm::AnyRef) +// - FuncRef: a two-word FunctionTableElem (wasm indirect call ABI) +// - FuncRef (if `isAsmJS`): a two-word FunctionTableElem (asm.js ABI) +// Eventually there should be a single unified AnyRef representation. + +struct TableDesc { + RefType elemType; + bool isImported; + bool isExported; + bool isAsmJS; + uint32_t initialLength; + Maybe maximumLength; + Maybe initExpr; + + TableDesc() = default; + TableDesc(RefType elemType, uint32_t initialLength, + Maybe maximumLength, Maybe&& initExpr, + bool isAsmJS, bool isImported = false, bool isExported = false) + : elemType(elemType), + isImported(isImported), + isExported(isExported), + isAsmJS(isAsmJS), + initialLength(initialLength), + maximumLength(maximumLength), + initExpr(std::move(initExpr)) {} +}; + +using TableDescVector = Vector; + +} // namespace wasm +} // namespace js + +#endif // wasm_module_types_h diff --git a/js/src/wasm/WasmOpIter.cpp b/js/src/wasm/WasmOpIter.cpp new file mode 100644 index 0000000000..91e2e17a70 --- /dev/null +++ b/js/src/wasm/WasmOpIter.cpp @@ -0,0 +1,863 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmOpIter.h" + +#include "jit/AtomicOp.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +#ifdef ENABLE_WASM_GC +# ifndef ENABLE_WASM_FUNCTION_REFERENCES +# error "GC types require the function-references feature" +# endif +#endif + +#ifdef DEBUG + +# ifdef ENABLE_WASM_FUNCTION_REFERENCES +# define WASM_FUNCTION_REFERENCES_OP(code) return code +# else +# define WASM_FUNCTION_REFERENCES_OP(code) break +# endif +# ifdef ENABLE_WASM_GC +# define WASM_GC_OP(code) return code +# else +# define WASM_GC_OP(code) break +# endif +# ifdef ENABLE_WASM_SIMD +# define WASM_SIMD_OP(code) return code +# else +# define WASM_SIMD_OP(code) break +# endif + +OpKind wasm::Classify(OpBytes op) { + switch (Op(op.b0)) { + case Op::Block: + return OpKind::Block; + case Op::Loop: + return OpKind::Loop; + case Op::Unreachable: + return OpKind::Unreachable; + case Op::Drop: + return OpKind::Drop; + case Op::I32Const: + return OpKind::I32; + case Op::I64Const: + return OpKind::I64; + case Op::F32Const: + return OpKind::F32; + case Op::F64Const: + return OpKind::F64; + case Op::Br: + return OpKind::Br; + case Op::BrIf: + return OpKind::BrIf; + case Op::BrTable: + return OpKind::BrTable; + case Op::Nop: + return OpKind::Nop; + case Op::I32Clz: + case Op::I32Ctz: + case Op::I32Popcnt: + case Op::I64Clz: + case Op::I64Ctz: + case Op::I64Popcnt: + case Op::F32Abs: + case Op::F32Neg: + case Op::F32Ceil: + case Op::F32Floor: + case Op::F32Trunc: + case Op::F32Nearest: + case Op::F32Sqrt: + case Op::F64Abs: + case Op::F64Neg: + case Op::F64Ceil: + case Op::F64Floor: + case Op::F64Trunc: + case Op::F64Nearest: + case Op::F64Sqrt: + return OpKind::Unary; + case Op::I32Add: + case Op::I32Sub: + case Op::I32Mul: + case Op::I32DivS: + case Op::I32DivU: + case Op::I32RemS: + case Op::I32RemU: + case Op::I32And: + case Op::I32Or: + case Op::I32Xor: + case Op::I32Shl: + case Op::I32ShrS: + case Op::I32ShrU: + case Op::I32Rotl: + case Op::I32Rotr: + case Op::I64Add: + case Op::I64Sub: + case Op::I64Mul: + case Op::I64DivS: + case Op::I64DivU: + case Op::I64RemS: + case Op::I64RemU: + case Op::I64And: + case Op::I64Or: + case Op::I64Xor: + case Op::I64Shl: + case Op::I64ShrS: + case Op::I64ShrU: + case Op::I64Rotl: + case Op::I64Rotr: + case Op::F32Add: + case Op::F32Sub: + case Op::F32Mul: + case Op::F32Div: + case Op::F32Min: + case Op::F32Max: + case Op::F32CopySign: + case Op::F64Add: + case Op::F64Sub: + case Op::F64Mul: + case Op::F64Div: + case Op::F64Min: + case Op::F64Max: + case Op::F64CopySign: + return OpKind::Binary; + case Op::I32Eq: + case Op::I32Ne: + case Op::I32LtS: + case Op::I32LtU: + case Op::I32LeS: + case Op::I32LeU: + case Op::I32GtS: + case Op::I32GtU: + case Op::I32GeS: + case Op::I32GeU: + case Op::I64Eq: + case Op::I64Ne: + case Op::I64LtS: + case Op::I64LtU: + case Op::I64LeS: + case Op::I64LeU: + case Op::I64GtS: + case Op::I64GtU: + case Op::I64GeS: + case Op::I64GeU: + case Op::F32Eq: + case Op::F32Ne: + case Op::F32Lt: + case Op::F32Le: + case Op::F32Gt: + case Op::F32Ge: + case Op::F64Eq: + case Op::F64Ne: + case Op::F64Lt: + case Op::F64Le: + case Op::F64Gt: + case Op::F64Ge: + return OpKind::Comparison; + case Op::I32Eqz: + case Op::I32WrapI64: + case Op::I32TruncF32S: + case Op::I32TruncF32U: + case Op::I32ReinterpretF32: + case Op::I32TruncF64S: + case Op::I32TruncF64U: + case Op::I64ExtendI32S: + case Op::I64ExtendI32U: + case Op::I64TruncF32S: + case Op::I64TruncF32U: + case Op::I64TruncF64S: + case Op::I64TruncF64U: + case Op::I64ReinterpretF64: + case Op::I64Eqz: + case Op::F32ConvertI32S: + case Op::F32ConvertI32U: + case Op::F32ReinterpretI32: + case Op::F32ConvertI64S: + case Op::F32ConvertI64U: + case Op::F32DemoteF64: + case Op::F64ConvertI32S: + case Op::F64ConvertI32U: + case Op::F64ConvertI64S: + case Op::F64ConvertI64U: + case Op::F64ReinterpretI64: + case Op::F64PromoteF32: + case Op::I32Extend8S: + case Op::I32Extend16S: + case Op::I64Extend8S: + case Op::I64Extend16S: + case Op::I64Extend32S: + return OpKind::Conversion; + case Op::I32Load8S: + case Op::I32Load8U: + case Op::I32Load16S: + case Op::I32Load16U: + case Op::I64Load8S: + case Op::I64Load8U: + case Op::I64Load16S: + case Op::I64Load16U: + case Op::I64Load32S: + case Op::I64Load32U: + case Op::I32Load: + case Op::I64Load: + case Op::F32Load: + case Op::F64Load: + return OpKind::Load; + case Op::I32Store8: + case Op::I32Store16: + case Op::I64Store8: + case Op::I64Store16: + case Op::I64Store32: + case Op::I32Store: + case Op::I64Store: + case Op::F32Store: + case Op::F64Store: + return OpKind::Store; + case Op::SelectNumeric: + case Op::SelectTyped: + return OpKind::Select; + case Op::LocalGet: + return OpKind::GetLocal; + case Op::LocalSet: + return OpKind::SetLocal; + case Op::LocalTee: + return OpKind::TeeLocal; + case Op::GlobalGet: + return OpKind::GetGlobal; + case Op::GlobalSet: + return OpKind::SetGlobal; + case Op::TableGet: + return OpKind::TableGet; + case Op::TableSet: + return OpKind::TableSet; + case Op::Call: + return OpKind::Call; + case Op::CallIndirect: + return OpKind::CallIndirect; + case Op::CallRef: + WASM_FUNCTION_REFERENCES_OP(OpKind::CallRef); + case Op::Return: + case Op::Limit: + // Accept Limit, for use in decoding the end of a function after the body. + return OpKind::Return; + case Op::If: + return OpKind::If; + case Op::Else: + return OpKind::Else; + case Op::End: + return OpKind::End; + case Op::Catch: + return OpKind::Catch; + case Op::CatchAll: + return OpKind::CatchAll; + case Op::Delegate: + return OpKind::Delegate; + case Op::Throw: + return OpKind::Throw; + case Op::Rethrow: + return OpKind::Rethrow; + case Op::Try: + return OpKind::Try; + case Op::MemorySize: + return OpKind::MemorySize; + case Op::MemoryGrow: + return OpKind::MemoryGrow; + case Op::RefNull: + return OpKind::RefNull; + case Op::RefIsNull: + return OpKind::Conversion; + case Op::RefFunc: + return OpKind::RefFunc; + case Op::RefAsNonNull: + WASM_FUNCTION_REFERENCES_OP(OpKind::RefAsNonNull); + case Op::BrOnNull: + WASM_FUNCTION_REFERENCES_OP(OpKind::BrOnNull); + case Op::BrOnNonNull: + WASM_FUNCTION_REFERENCES_OP(OpKind::BrOnNonNull); + case Op::RefEq: + WASM_GC_OP(OpKind::Comparison); + case Op::GcPrefix: { + switch (GcOp(op.b1)) { + case GcOp::Limit: + // Reject Limit for GcPrefix encoding + break; + case GcOp::StructNew: + WASM_GC_OP(OpKind::StructNew); + case GcOp::StructNewDefault: + WASM_GC_OP(OpKind::StructNewDefault); + case GcOp::StructGet: + case GcOp::StructGetS: + case GcOp::StructGetU: + WASM_GC_OP(OpKind::StructGet); + case GcOp::StructSet: + WASM_GC_OP(OpKind::StructSet); + case GcOp::ArrayNew: + WASM_GC_OP(OpKind::ArrayNew); + case GcOp::ArrayNewFixed: + WASM_GC_OP(OpKind::ArrayNewFixed); + case GcOp::ArrayNewDefault: + WASM_GC_OP(OpKind::ArrayNewDefault); + case GcOp::ArrayNewData: + case GcOp::ArrayInitFromElemStaticV5: + case GcOp::ArrayNewElem: + WASM_GC_OP(OpKind::ArrayNewData); + case GcOp::ArrayGet: + case GcOp::ArrayGetS: + case GcOp::ArrayGetU: + WASM_GC_OP(OpKind::ArrayGet); + case GcOp::ArraySet: + WASM_GC_OP(OpKind::ArraySet); + case GcOp::ArrayLenWithTypeIndex: + case GcOp::ArrayLen: + WASM_GC_OP(OpKind::ArrayLen); + case GcOp::ArrayCopy: + WASM_GC_OP(OpKind::ArrayCopy); + case GcOp::RefTestV5: + WASM_GC_OP(OpKind::RefTestV5); + case GcOp::RefCastV5: + WASM_GC_OP(OpKind::RefCastV5); + case GcOp::RefTest: + case GcOp::RefTestNull: + WASM_GC_OP(OpKind::RefTest); + case GcOp::RefCast: + case GcOp::RefCastNull: + WASM_GC_OP(OpKind::RefCast); + case GcOp::BrOnCast: + WASM_GC_OP(OpKind::BrOnCast); + case GcOp::BrOnCastV5: + case GcOp::BrOnCastHeapV5: + case GcOp::BrOnCastHeapNullV5: + WASM_GC_OP(OpKind::BrOnCastV5); + case GcOp::BrOnCastFailV5: + case GcOp::BrOnCastFailHeapV5: + case GcOp::BrOnCastFailHeapNullV5: + WASM_GC_OP(OpKind::BrOnCastFailV5); + case GcOp::RefAsStructV5: + WASM_GC_OP(OpKind::Conversion); + case GcOp::BrOnNonStructV5: + WASM_GC_OP(OpKind::BrOnNonStructV5); + case GcOp::ExternInternalize: + WASM_GC_OP(OpKind::RefConversion); + case GcOp::ExternExternalize: + WASM_GC_OP(OpKind::RefConversion); + } + break; + } + case Op::SimdPrefix: { + switch (SimdOp(op.b1)) { + case SimdOp::MozPMADDUBSW: + case SimdOp::Limit: + // Reject Limit and reserved codes for SimdPrefix encoding + break; + case SimdOp::I8x16ExtractLaneS: + case SimdOp::I8x16ExtractLaneU: + case SimdOp::I16x8ExtractLaneS: + case SimdOp::I16x8ExtractLaneU: + case SimdOp::I32x4ExtractLane: + case SimdOp::I64x2ExtractLane: + case SimdOp::F32x4ExtractLane: + case SimdOp::F64x2ExtractLane: + WASM_SIMD_OP(OpKind::ExtractLane); + case SimdOp::I8x16Splat: + case SimdOp::I16x8Splat: + case SimdOp::I32x4Splat: + case SimdOp::I64x2Splat: + case SimdOp::F32x4Splat: + case SimdOp::F64x2Splat: + case SimdOp::V128AnyTrue: + case SimdOp::I8x16AllTrue: + case SimdOp::I16x8AllTrue: + case SimdOp::I32x4AllTrue: + case SimdOp::I64x2AllTrue: + case SimdOp::I8x16Bitmask: + case SimdOp::I16x8Bitmask: + case SimdOp::I32x4Bitmask: + case SimdOp::I64x2Bitmask: + WASM_SIMD_OP(OpKind::Conversion); + case SimdOp::I8x16ReplaceLane: + case SimdOp::I16x8ReplaceLane: + case SimdOp::I32x4ReplaceLane: + case SimdOp::I64x2ReplaceLane: + case SimdOp::F32x4ReplaceLane: + case SimdOp::F64x2ReplaceLane: + WASM_SIMD_OP(OpKind::ReplaceLane); + case SimdOp::I8x16Eq: + case SimdOp::I8x16Ne: + case SimdOp::I8x16LtS: + case SimdOp::I8x16LtU: + case SimdOp::I8x16GtS: + case SimdOp::I8x16GtU: + case SimdOp::I8x16LeS: + case SimdOp::I8x16LeU: + case SimdOp::I8x16GeS: + case SimdOp::I8x16GeU: + case SimdOp::I16x8Eq: + case SimdOp::I16x8Ne: + case SimdOp::I16x8LtS: + case SimdOp::I16x8LtU: + case SimdOp::I16x8GtS: + case SimdOp::I16x8GtU: + case SimdOp::I16x8LeS: + case SimdOp::I16x8LeU: + case SimdOp::I16x8GeS: + case SimdOp::I16x8GeU: + case SimdOp::I32x4Eq: + case SimdOp::I32x4Ne: + case SimdOp::I32x4LtS: + case SimdOp::I32x4LtU: + case SimdOp::I32x4GtS: + case SimdOp::I32x4GtU: + case SimdOp::I32x4LeS: + case SimdOp::I32x4LeU: + case SimdOp::I32x4GeS: + case SimdOp::I32x4GeU: + case SimdOp::I64x2Eq: + case SimdOp::I64x2Ne: + case SimdOp::I64x2LtS: + case SimdOp::I64x2GtS: + case SimdOp::I64x2LeS: + case SimdOp::I64x2GeS: + case SimdOp::F32x4Eq: + case SimdOp::F32x4Ne: + case SimdOp::F32x4Lt: + case SimdOp::F32x4Gt: + case SimdOp::F32x4Le: + case SimdOp::F32x4Ge: + case SimdOp::F64x2Eq: + case SimdOp::F64x2Ne: + case SimdOp::F64x2Lt: + case SimdOp::F64x2Gt: + case SimdOp::F64x2Le: + case SimdOp::F64x2Ge: + case SimdOp::V128And: + case SimdOp::V128Or: + case SimdOp::V128Xor: + case SimdOp::V128AndNot: + case SimdOp::I8x16AvgrU: + case SimdOp::I16x8AvgrU: + case SimdOp::I8x16Add: + case SimdOp::I8x16AddSatS: + case SimdOp::I8x16AddSatU: + case SimdOp::I8x16Sub: + case SimdOp::I8x16SubSatS: + case SimdOp::I8x16SubSatU: + case SimdOp::I8x16MinS: + case SimdOp::I8x16MaxS: + case SimdOp::I8x16MinU: + case SimdOp::I8x16MaxU: + case SimdOp::I16x8Add: + case SimdOp::I16x8AddSatS: + case SimdOp::I16x8AddSatU: + case SimdOp::I16x8Sub: + case SimdOp::I16x8SubSatS: + case SimdOp::I16x8SubSatU: + case SimdOp::I16x8Mul: + case SimdOp::I16x8MinS: + case SimdOp::I16x8MaxS: + case SimdOp::I16x8MinU: + case SimdOp::I16x8MaxU: + case SimdOp::I32x4Add: + case SimdOp::I32x4Sub: + case SimdOp::I32x4Mul: + case SimdOp::I32x4MinS: + case SimdOp::I32x4MaxS: + case SimdOp::I32x4MinU: + case SimdOp::I32x4MaxU: + case SimdOp::I64x2Add: + case SimdOp::I64x2Sub: + case SimdOp::I64x2Mul: + case SimdOp::F32x4Add: + case SimdOp::F32x4Sub: + case SimdOp::F32x4Mul: + case SimdOp::F32x4Div: + case SimdOp::F32x4Min: + case SimdOp::F32x4Max: + case SimdOp::F64x2Add: + case SimdOp::F64x2Sub: + case SimdOp::F64x2Mul: + case SimdOp::F64x2Div: + case SimdOp::F64x2Min: + case SimdOp::F64x2Max: + case SimdOp::I8x16NarrowI16x8S: + case SimdOp::I8x16NarrowI16x8U: + case SimdOp::I16x8NarrowI32x4S: + case SimdOp::I16x8NarrowI32x4U: + case SimdOp::I8x16Swizzle: + case SimdOp::F32x4PMin: + case SimdOp::F32x4PMax: + case SimdOp::F64x2PMin: + case SimdOp::F64x2PMax: + case SimdOp::I32x4DotI16x8S: + case SimdOp::I16x8ExtmulLowI8x16S: + case SimdOp::I16x8ExtmulHighI8x16S: + case SimdOp::I16x8ExtmulLowI8x16U: + case SimdOp::I16x8ExtmulHighI8x16U: + case SimdOp::I32x4ExtmulLowI16x8S: + case SimdOp::I32x4ExtmulHighI16x8S: + case SimdOp::I32x4ExtmulLowI16x8U: + case SimdOp::I32x4ExtmulHighI16x8U: + case SimdOp::I64x2ExtmulLowI32x4S: + case SimdOp::I64x2ExtmulHighI32x4S: + case SimdOp::I64x2ExtmulLowI32x4U: + case SimdOp::I64x2ExtmulHighI32x4U: + case SimdOp::I16x8Q15MulrSatS: + case SimdOp::F32x4RelaxedMin: + case SimdOp::F32x4RelaxedMax: + case SimdOp::F64x2RelaxedMin: + case SimdOp::F64x2RelaxedMax: + case SimdOp::I8x16RelaxedSwizzle: + case SimdOp::I16x8RelaxedQ15MulrS: + case SimdOp::I16x8DotI8x16I7x16S: + WASM_SIMD_OP(OpKind::Binary); + case SimdOp::I8x16Neg: + case SimdOp::I16x8Neg: + case SimdOp::I16x8ExtendLowI8x16S: + case SimdOp::I16x8ExtendHighI8x16S: + case SimdOp::I16x8ExtendLowI8x16U: + case SimdOp::I16x8ExtendHighI8x16U: + case SimdOp::I32x4Neg: + case SimdOp::I32x4ExtendLowI16x8S: + case SimdOp::I32x4ExtendHighI16x8S: + case SimdOp::I32x4ExtendLowI16x8U: + case SimdOp::I32x4ExtendHighI16x8U: + case SimdOp::I32x4TruncSatF32x4S: + case SimdOp::I32x4TruncSatF32x4U: + case SimdOp::I64x2Neg: + case SimdOp::I64x2ExtendLowI32x4S: + case SimdOp::I64x2ExtendHighI32x4S: + case SimdOp::I64x2ExtendLowI32x4U: + case SimdOp::I64x2ExtendHighI32x4U: + case SimdOp::F32x4Abs: + case SimdOp::F32x4Neg: + case SimdOp::F32x4Sqrt: + case SimdOp::F32x4ConvertI32x4S: + case SimdOp::F32x4ConvertI32x4U: + case SimdOp::F64x2Abs: + case SimdOp::F64x2Neg: + case SimdOp::F64x2Sqrt: + case SimdOp::V128Not: + case SimdOp::I8x16Popcnt: + case SimdOp::I8x16Abs: + case SimdOp::I16x8Abs: + case SimdOp::I32x4Abs: + case SimdOp::I64x2Abs: + case SimdOp::F32x4Ceil: + case SimdOp::F32x4Floor: + case SimdOp::F32x4Trunc: + case SimdOp::F32x4Nearest: + case SimdOp::F64x2Ceil: + case SimdOp::F64x2Floor: + case SimdOp::F64x2Trunc: + case SimdOp::F64x2Nearest: + case SimdOp::F32x4DemoteF64x2Zero: + case SimdOp::F64x2PromoteLowF32x4: + case SimdOp::F64x2ConvertLowI32x4S: + case SimdOp::F64x2ConvertLowI32x4U: + case SimdOp::I32x4TruncSatF64x2SZero: + case SimdOp::I32x4TruncSatF64x2UZero: + case SimdOp::I16x8ExtaddPairwiseI8x16S: + case SimdOp::I16x8ExtaddPairwiseI8x16U: + case SimdOp::I32x4ExtaddPairwiseI16x8S: + case SimdOp::I32x4ExtaddPairwiseI16x8U: + case SimdOp::I32x4RelaxedTruncF32x4S: + case SimdOp::I32x4RelaxedTruncF32x4U: + case SimdOp::I32x4RelaxedTruncF64x2SZero: + case SimdOp::I32x4RelaxedTruncF64x2UZero: + WASM_SIMD_OP(OpKind::Unary); + case SimdOp::I8x16Shl: + case SimdOp::I8x16ShrS: + case SimdOp::I8x16ShrU: + case SimdOp::I16x8Shl: + case SimdOp::I16x8ShrS: + case SimdOp::I16x8ShrU: + case SimdOp::I32x4Shl: + case SimdOp::I32x4ShrS: + case SimdOp::I32x4ShrU: + case SimdOp::I64x2Shl: + case SimdOp::I64x2ShrS: + case SimdOp::I64x2ShrU: + WASM_SIMD_OP(OpKind::VectorShift); + case SimdOp::V128Bitselect: + WASM_SIMD_OP(OpKind::Ternary); + case SimdOp::I8x16Shuffle: + WASM_SIMD_OP(OpKind::VectorShuffle); + case SimdOp::V128Const: + WASM_SIMD_OP(OpKind::V128); + case SimdOp::V128Load: + case SimdOp::V128Load8Splat: + case SimdOp::V128Load16Splat: + case SimdOp::V128Load32Splat: + case SimdOp::V128Load64Splat: + case SimdOp::V128Load8x8S: + case SimdOp::V128Load8x8U: + case SimdOp::V128Load16x4S: + case SimdOp::V128Load16x4U: + case SimdOp::V128Load32x2S: + case SimdOp::V128Load32x2U: + case SimdOp::V128Load32Zero: + case SimdOp::V128Load64Zero: + WASM_SIMD_OP(OpKind::Load); + case SimdOp::V128Store: + WASM_SIMD_OP(OpKind::Store); + case SimdOp::V128Load8Lane: + case SimdOp::V128Load16Lane: + case SimdOp::V128Load32Lane: + case SimdOp::V128Load64Lane: + WASM_SIMD_OP(OpKind::LoadLane); + case SimdOp::V128Store8Lane: + case SimdOp::V128Store16Lane: + case SimdOp::V128Store32Lane: + case SimdOp::V128Store64Lane: + WASM_SIMD_OP(OpKind::StoreLane); + case SimdOp::F32x4RelaxedFma: + case SimdOp::F32x4RelaxedFnma: + case SimdOp::F64x2RelaxedFma: + case SimdOp::F64x2RelaxedFnma: + case SimdOp::I8x16RelaxedLaneSelect: + case SimdOp::I16x8RelaxedLaneSelect: + case SimdOp::I32x4RelaxedLaneSelect: + case SimdOp::I64x2RelaxedLaneSelect: + case SimdOp::I32x4DotI8x16I7x16AddS: + WASM_SIMD_OP(OpKind::Ternary); + } + break; + } + case Op::MiscPrefix: { + switch (MiscOp(op.b1)) { + case MiscOp::Limit: + // Reject Limit for MiscPrefix encoding + break; + case MiscOp::I32TruncSatF32S: + case MiscOp::I32TruncSatF32U: + case MiscOp::I32TruncSatF64S: + case MiscOp::I32TruncSatF64U: + case MiscOp::I64TruncSatF32S: + case MiscOp::I64TruncSatF32U: + case MiscOp::I64TruncSatF64S: + case MiscOp::I64TruncSatF64U: + return OpKind::Conversion; + case MiscOp::MemoryCopy: + case MiscOp::TableCopy: + return OpKind::MemOrTableCopy; + case MiscOp::DataDrop: + case MiscOp::ElemDrop: + return OpKind::DataOrElemDrop; + case MiscOp::MemoryFill: + return OpKind::MemFill; + case MiscOp::MemoryInit: + case MiscOp::TableInit: + return OpKind::MemOrTableInit; + case MiscOp::TableFill: + return OpKind::TableFill; + case MiscOp::MemoryDiscard: + return OpKind::MemDiscard; + case MiscOp::TableGrow: + return OpKind::TableGrow; + case MiscOp::TableSize: + return OpKind::TableSize; + } + break; + } + case Op::ThreadPrefix: { + switch (ThreadOp(op.b1)) { + case ThreadOp::Limit: + // Reject Limit for ThreadPrefix encoding + break; + case ThreadOp::Wake: + return OpKind::Wake; + case ThreadOp::I32Wait: + case ThreadOp::I64Wait: + return OpKind::Wait; + case ThreadOp::Fence: + return OpKind::Fence; + case ThreadOp::I32AtomicLoad: + case ThreadOp::I64AtomicLoad: + case ThreadOp::I32AtomicLoad8U: + case ThreadOp::I32AtomicLoad16U: + case ThreadOp::I64AtomicLoad8U: + case ThreadOp::I64AtomicLoad16U: + case ThreadOp::I64AtomicLoad32U: + return OpKind::AtomicLoad; + case ThreadOp::I32AtomicStore: + case ThreadOp::I64AtomicStore: + case ThreadOp::I32AtomicStore8U: + case ThreadOp::I32AtomicStore16U: + case ThreadOp::I64AtomicStore8U: + case ThreadOp::I64AtomicStore16U: + case ThreadOp::I64AtomicStore32U: + return OpKind::AtomicStore; + case ThreadOp::I32AtomicAdd: + case ThreadOp::I64AtomicAdd: + case ThreadOp::I32AtomicAdd8U: + case ThreadOp::I32AtomicAdd16U: + case ThreadOp::I64AtomicAdd8U: + case ThreadOp::I64AtomicAdd16U: + case ThreadOp::I64AtomicAdd32U: + case ThreadOp::I32AtomicSub: + case ThreadOp::I64AtomicSub: + case ThreadOp::I32AtomicSub8U: + case ThreadOp::I32AtomicSub16U: + case ThreadOp::I64AtomicSub8U: + case ThreadOp::I64AtomicSub16U: + case ThreadOp::I64AtomicSub32U: + case ThreadOp::I32AtomicAnd: + case ThreadOp::I64AtomicAnd: + case ThreadOp::I32AtomicAnd8U: + case ThreadOp::I32AtomicAnd16U: + case ThreadOp::I64AtomicAnd8U: + case ThreadOp::I64AtomicAnd16U: + case ThreadOp::I64AtomicAnd32U: + case ThreadOp::I32AtomicOr: + case ThreadOp::I64AtomicOr: + case ThreadOp::I32AtomicOr8U: + case ThreadOp::I32AtomicOr16U: + case ThreadOp::I64AtomicOr8U: + case ThreadOp::I64AtomicOr16U: + case ThreadOp::I64AtomicOr32U: + case ThreadOp::I32AtomicXor: + case ThreadOp::I64AtomicXor: + case ThreadOp::I32AtomicXor8U: + case ThreadOp::I32AtomicXor16U: + case ThreadOp::I64AtomicXor8U: + case ThreadOp::I64AtomicXor16U: + case ThreadOp::I64AtomicXor32U: + case ThreadOp::I32AtomicXchg: + case ThreadOp::I64AtomicXchg: + case ThreadOp::I32AtomicXchg8U: + case ThreadOp::I32AtomicXchg16U: + case ThreadOp::I64AtomicXchg8U: + case ThreadOp::I64AtomicXchg16U: + case ThreadOp::I64AtomicXchg32U: + return OpKind::AtomicBinOp; + case ThreadOp::I32AtomicCmpXchg: + case ThreadOp::I64AtomicCmpXchg: + case ThreadOp::I32AtomicCmpXchg8U: + case ThreadOp::I32AtomicCmpXchg16U: + case ThreadOp::I64AtomicCmpXchg8U: + case ThreadOp::I64AtomicCmpXchg16U: + case ThreadOp::I64AtomicCmpXchg32U: + return OpKind::AtomicCompareExchange; + default: + break; + } + break; + } + case Op::MozPrefix: { + switch (MozOp(op.b1)) { + case MozOp::Limit: + // Reject Limit for the MozPrefix encoding + break; + case MozOp::TeeGlobal: + return OpKind::TeeGlobal; + case MozOp::I32BitNot: + case MozOp::I32Abs: + case MozOp::I32Neg: + return OpKind::Unary; + case MozOp::I32Min: + case MozOp::I32Max: + case MozOp::F64Mod: + case MozOp::F64Pow: + case MozOp::F64Atan2: + return OpKind::Binary; + case MozOp::F64SinNative: + case MozOp::F64SinFdlibm: + case MozOp::F64CosNative: + case MozOp::F64CosFdlibm: + case MozOp::F64TanNative: + case MozOp::F64TanFdlibm: + case MozOp::F64Asin: + case MozOp::F64Acos: + case MozOp::F64Atan: + case MozOp::F64Exp: + case MozOp::F64Log: + return OpKind::Unary; + case MozOp::I32TeeStore8: + case MozOp::I32TeeStore16: + case MozOp::I64TeeStore8: + case MozOp::I64TeeStore16: + case MozOp::I64TeeStore32: + case MozOp::I32TeeStore: + case MozOp::I64TeeStore: + case MozOp::F32TeeStore: + case MozOp::F64TeeStore: + case MozOp::F32TeeStoreF64: + case MozOp::F64TeeStoreF32: + return OpKind::TeeStore; + case MozOp::OldCallDirect: + return OpKind::OldCallDirect; + case MozOp::OldCallIndirect: + return OpKind::OldCallIndirect; + case MozOp::Intrinsic: + return OpKind::Intrinsic; + } + break; + } + case Op::FirstPrefix: + break; + } + MOZ_CRASH("unimplemented opcode"); +} + +# undef WASM_GC_OP +# undef WASM_REF_OP + +#endif // DEBUG + +bool UnsetLocalsState::init(const ValTypeVector& locals, size_t numParams) { + MOZ_ASSERT(setLocalsStack_.empty()); + + // Find the first and total count of non-defaultable locals. + size_t firstNonDefaultable = UINT32_MAX; + size_t countNonDefaultable = 0; + for (size_t i = numParams; i < locals.length(); i++) { + if (!locals[i].isDefaultable()) { + firstNonDefaultable = std::min(i, firstNonDefaultable); + countNonDefaultable++; + } + } + firstNonDefaultLocal_ = firstNonDefaultable; + if (countNonDefaultable == 0) { + // No locals to track, saving CPU cycles. + MOZ_ASSERT(firstNonDefaultable == UINT32_MAX); + return true; + } + + // setLocalsStack_ cannot be deeper than amount of non-defaultable locals. + if (!setLocalsStack_.reserve(countNonDefaultable)) { + return false; + } + + // Allocate a bitmap for locals starting at the first non-defaultable local. + size_t bitmapSize = + ((locals.length() - firstNonDefaultable) + (WordBits - 1)) / WordBits; + if (!unsetLocals_.resize(bitmapSize)) { + return false; + } + memset(unsetLocals_.begin(), 0, bitmapSize * WordSize); + for (size_t i = firstNonDefaultable; i < locals.length(); i++) { + if (!locals[i].isDefaultable()) { + size_t localUnsetIndex = i - firstNonDefaultable; + unsetLocals_[localUnsetIndex / WordBits] |= + 1 << (localUnsetIndex % WordBits); + } + } + return true; +} diff --git a/js/src/wasm/WasmOpIter.h b/js/src/wasm/WasmOpIter.h new file mode 100644 index 0000000000..c175921dc9 --- /dev/null +++ b/js/src/wasm/WasmOpIter.h @@ -0,0 +1,4239 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_op_iter_h +#define wasm_op_iter_h + +#include "mozilla/CompactPair.h" +#include "mozilla/Poison.h" + +#include + +#include "js/Printf.h" +#include "wasm/WasmIntrinsic.h" +#include "wasm/WasmUtility.h" +#include "wasm/WasmValidate.h" + +namespace js { +namespace wasm { + +// The kind of a control-flow stack item. +enum class LabelKind : uint8_t { + Body, + Block, + Loop, + Then, + Else, + Try, + Catch, + CatchAll, +}; + +// The type of values on the operand stack during validation. This is either a +// ValType or the special type "Bottom". + +class StackType { + PackedTypeCode tc_; + + explicit StackType(PackedTypeCode tc) : tc_(tc) {} + + public: + StackType() : tc_(PackedTypeCode::invalid()) {} + + explicit StackType(const ValType& t) : tc_(t.packed()) { + MOZ_ASSERT(tc_.isValid()); + MOZ_ASSERT(!isStackBottom()); + } + + static StackType bottom() { + return StackType(PackedTypeCode::pack(TypeCode::Limit)); + } + + bool isStackBottom() const { + MOZ_ASSERT(tc_.isValid()); + return tc_.typeCode() == TypeCode::Limit; + } + + // Returns whether this input is nullable when interpreted as an operand. + // When the type is bottom for unreachable code, this returns false as that + // is the most permissive option. + bool isNullableAsOperand() const { + MOZ_ASSERT(tc_.isValid()); + return isStackBottom() ? false : tc_.isNullable(); + } + + ValType valType() const { + MOZ_ASSERT(tc_.isValid()); + MOZ_ASSERT(!isStackBottom()); + return ValType(tc_); + } + + ValType valTypeOr(ValType ifBottom) const { + MOZ_ASSERT(tc_.isValid()); + if (isStackBottom()) { + return ifBottom; + } + return valType(); + } + + ValType asNonNullable() const { + MOZ_ASSERT(tc_.isValid()); + MOZ_ASSERT(!isStackBottom()); + return ValType(tc_.withIsNullable(false)); + } + + bool isValidForUntypedSelect() const { + MOZ_ASSERT(tc_.isValid()); + if (isStackBottom()) { + return true; + } + switch (valType().kind()) { + case ValType::I32: + case ValType::F32: + case ValType::I64: + case ValType::F64: +#ifdef ENABLE_WASM_SIMD + case ValType::V128: +#endif + return true; + default: + return false; + } + } + + bool operator==(const StackType& that) const { + MOZ_ASSERT(tc_.isValid() && that.tc_.isValid()); + return tc_ == that.tc_; + } + + bool operator!=(const StackType& that) const { + MOZ_ASSERT(tc_.isValid() && that.tc_.isValid()); + return tc_ != that.tc_; + } +}; + +#ifdef DEBUG +// Families of opcodes that share a signature and validation logic. +enum class OpKind { + Block, + Loop, + Unreachable, + Drop, + I32, + I64, + F32, + F64, + V128, + Br, + BrIf, + BrTable, + Nop, + Unary, + Binary, + Ternary, + Comparison, + Conversion, + Load, + Store, + TeeStore, + MemorySize, + MemoryGrow, + Select, + GetLocal, + SetLocal, + TeeLocal, + GetGlobal, + SetGlobal, + TeeGlobal, + Call, + CallIndirect, +# ifdef ENABLE_WASM_FUNCTION_REFERENCES + CallRef, +# endif + OldCallDirect, + OldCallIndirect, + Return, + If, + Else, + End, + Wait, + Wake, + Fence, + AtomicLoad, + AtomicStore, + AtomicBinOp, + AtomicCompareExchange, + MemOrTableCopy, + DataOrElemDrop, + MemFill, + MemOrTableInit, + TableFill, + MemDiscard, + TableGet, + TableGrow, + TableSet, + TableSize, + RefNull, + RefFunc, + RefAsNonNull, + BrOnNull, + BrOnNonNull, + StructNew, + StructNewDefault, + StructGet, + StructSet, + ArrayNew, + ArrayNewFixed, + ArrayNewDefault, + ArrayNewData, + ArrayNewElem, + ArrayGet, + ArraySet, + ArrayLen, + ArrayCopy, + RefTestV5, + RefCastV5, + BrOnCastV5, + BrOnCastFailV5, + BrOnNonStructV5, + RefTest, + RefCast, + BrOnCast, + RefConversion, +# ifdef ENABLE_WASM_SIMD + ExtractLane, + ReplaceLane, + LoadLane, + StoreLane, + VectorShift, + VectorShuffle, +# endif + Catch, + CatchAll, + Delegate, + Throw, + Rethrow, + Try, + Intrinsic, +}; + +// Return the OpKind for a given Op. This is used for sanity-checking that +// API users use the correct read function for a given Op. +OpKind Classify(OpBytes op); +#endif + +// Common fields for linear memory access. +template +struct LinearMemoryAddress { + Value base; + uint64_t offset; + uint32_t align; + + LinearMemoryAddress() : offset(0), align(0) {} + LinearMemoryAddress(Value base, uint64_t offset, uint32_t align) + : base(base), offset(offset), align(align) {} +}; + +template +class ControlStackEntry { + // Use a pair to optimize away empty ControlItem. + mozilla::CompactPair typeAndItem_; + + // The "base" of a control stack entry is valueStack_.length() minus + // type().params().length(), i.e., the size of the value stack "below" + // this block. + uint32_t valueStackBase_; + bool polymorphicBase_; + + LabelKind kind_; + + public: + ControlStackEntry(LabelKind kind, BlockType type, uint32_t valueStackBase) + : typeAndItem_(type, ControlItem()), + valueStackBase_(valueStackBase), + polymorphicBase_(false), + kind_(kind) { + MOZ_ASSERT(type != BlockType()); + } + + LabelKind kind() const { return kind_; } + BlockType type() const { return typeAndItem_.first(); } + ResultType resultType() const { return type().results(); } + ResultType branchTargetType() const { + return kind_ == LabelKind::Loop ? type().params() : type().results(); + } + uint32_t valueStackBase() const { return valueStackBase_; } + ControlItem& controlItem() { return typeAndItem_.second(); } + void setPolymorphicBase() { polymorphicBase_ = true; } + bool polymorphicBase() const { return polymorphicBase_; } + + void switchToElse() { + MOZ_ASSERT(kind() == LabelKind::Then); + kind_ = LabelKind::Else; + polymorphicBase_ = false; + } + + void switchToCatch() { + MOZ_ASSERT(kind() == LabelKind::Try || kind() == LabelKind::Catch); + kind_ = LabelKind::Catch; + polymorphicBase_ = false; + } + + void switchToCatchAll() { + MOZ_ASSERT(kind() == LabelKind::Try || kind() == LabelKind::Catch); + kind_ = LabelKind::CatchAll; + polymorphicBase_ = false; + } +}; + +// Track state of the non-defaultable locals. Every time such local is +// initialized, the stack will record at what depth and which local was set. +// On a block end, the "unset" state will be rolled back to how it was before +// the block started. +// +// It is very likely only a few functions will have non-defaultable locals and +// very few locals will be non-defaultable. This class is optimized to be fast +// for this common case. +class UnsetLocalsState { + struct SetLocalEntry { + uint32_t depth; + uint32_t localUnsetIndex; + SetLocalEntry(uint32_t depth_, uint32_t localUnsetIndex_) + : depth(depth_), localUnsetIndex(localUnsetIndex_) {} + }; + using SetLocalsStack = Vector; + using UnsetLocals = Vector; + + static constexpr size_t WordSize = 4; + static constexpr size_t WordBits = WordSize * 8; + + // Bit array of "unset" function locals. Stores only unset states of the + // locals that are declared after the first non-defaultable local. + UnsetLocals unsetLocals_; + // Stack of "set" operations. Contains pair where the first field is a depth, + // and the second field is local id (offset by firstNonDefaultLocal_). + SetLocalsStack setLocalsStack_; + uint32_t firstNonDefaultLocal_; + + public: + UnsetLocalsState() : firstNonDefaultLocal_(UINT32_MAX) {} + + [[nodiscard]] bool init(const ValTypeVector& locals, size_t numParams); + + inline bool isUnset(uint32_t id) const { + if (MOZ_LIKELY(id < firstNonDefaultLocal_)) { + return false; + } + uint32_t localUnsetIndex = id - firstNonDefaultLocal_; + return unsetLocals_[localUnsetIndex / WordBits] & + (1 << (localUnsetIndex % WordBits)); + } + + inline void set(uint32_t id, uint32_t depth) { + MOZ_ASSERT(isUnset(id)); + MOZ_ASSERT(id >= firstNonDefaultLocal_ && + (id - firstNonDefaultLocal_) / WordBits < unsetLocals_.length()); + uint32_t localUnsetIndex = id - firstNonDefaultLocal_; + unsetLocals_[localUnsetIndex / WordBits] ^= 1 + << (localUnsetIndex % WordBits); + // The setLocalsStack_ is reserved upfront in the UnsetLocalsState::init. + // A SetLocalEntry will be pushed only once per local. + setLocalsStack_.infallibleEmplaceBack(depth, localUnsetIndex); + } + + inline void resetToBlock(uint32_t controlDepth) { + while (MOZ_UNLIKELY(setLocalsStack_.length() > 0) && + setLocalsStack_.back().depth > controlDepth) { + uint32_t localUnsetIndex = setLocalsStack_.back().localUnsetIndex; + MOZ_ASSERT(!(unsetLocals_[localUnsetIndex / WordBits] & + (1 << (localUnsetIndex % WordBits)))); + unsetLocals_[localUnsetIndex / WordBits] |= + 1 << (localUnsetIndex % WordBits); + setLocalsStack_.popBack(); + } + } + + int empty() const { return setLocalsStack_.empty(); } +}; + +template +class TypeAndValueT { + // Use a Pair to optimize away empty Value. + mozilla::CompactPair tv_; + + public: + TypeAndValueT() : tv_(StackType::bottom(), Value()) {} + explicit TypeAndValueT(StackType type) : tv_(type, Value()) {} + explicit TypeAndValueT(ValType type) : tv_(StackType(type), Value()) {} + TypeAndValueT(StackType type, Value value) : tv_(type, value) {} + TypeAndValueT(ValType type, Value value) : tv_(StackType(type), value) {} + StackType type() const { return tv_.first(); } + void setType(StackType type) { tv_.first() = type; } + Value value() const { return tv_.second(); } + void setValue(Value value) { tv_.second() = value; } +}; + +// An iterator over the bytes of a function body. It performs validation +// and unpacks the data into a usable form. +// +// The MOZ_STACK_CLASS attribute here is because of the use of DebugOnly. +// There's otherwise nothing inherent in this class which would require +// it to be used on the stack. +template +class MOZ_STACK_CLASS OpIter : private Policy { + public: + using Value = typename Policy::Value; + using ValueVector = typename Policy::ValueVector; + using TypeAndValue = TypeAndValueT; + using TypeAndValueStack = Vector; + using ControlItem = typename Policy::ControlItem; + using Control = ControlStackEntry; + using ControlStack = Vector; + + enum Kind { + Func, + InitExpr, + }; + + private: + Kind kind_; + Decoder& d_; + const ModuleEnvironment& env_; + + TypeAndValueStack valueStack_; + TypeAndValueStack elseParamStack_; + ControlStack controlStack_; + UnsetLocalsState unsetLocals_; + // The exclusive max index of a global that can be accessed by global.get in + // this expression. When GC is enabled, this is any previously defined + // global. Otherwise this is always set to zero, and only imported immutable + // globals are allowed. + uint32_t maxInitializedGlobalsIndexPlus1_; + +#ifdef DEBUG + OpBytes op_; +#endif + size_t offsetOfLastReadOp_; + + [[nodiscard]] bool readFixedU8(uint8_t* out) { return d_.readFixedU8(out); } + [[nodiscard]] bool readFixedU32(uint32_t* out) { + return d_.readFixedU32(out); + } + [[nodiscard]] bool readVarS32(int32_t* out) { return d_.readVarS32(out); } + [[nodiscard]] bool readVarU32(uint32_t* out) { return d_.readVarU32(out); } + [[nodiscard]] bool readVarS64(int64_t* out) { return d_.readVarS64(out); } + [[nodiscard]] bool readVarU64(uint64_t* out) { return d_.readVarU64(out); } + [[nodiscard]] bool readFixedF32(float* out) { return d_.readFixedF32(out); } + [[nodiscard]] bool readFixedF64(double* out) { return d_.readFixedF64(out); } + + [[nodiscard]] bool readMemOrTableIndex(bool isMem, uint32_t* index); + [[nodiscard]] bool readLinearMemoryAddress(uint32_t byteSize, + LinearMemoryAddress* addr); + [[nodiscard]] bool readLinearMemoryAddressAligned( + uint32_t byteSize, LinearMemoryAddress* addr); + [[nodiscard]] bool readBlockType(BlockType* type); + [[nodiscard]] bool readGcTypeIndex(uint32_t* typeIndex); + [[nodiscard]] bool readStructTypeIndex(uint32_t* typeIndex); + [[nodiscard]] bool readArrayTypeIndex(uint32_t* typeIndex); + [[nodiscard]] bool readFuncTypeIndex(uint32_t* typeIndex); + [[nodiscard]] bool readFieldIndex(uint32_t* fieldIndex, + const StructType& structType); + + [[nodiscard]] bool popCallArgs(const ValTypeVector& expectedTypes, + ValueVector* values); + + [[nodiscard]] bool failEmptyStack(); + [[nodiscard]] bool popStackType(StackType* type, Value* value); + [[nodiscard]] bool popWithType(ValType expected, Value* value, + StackType* stackType); + [[nodiscard]] bool popWithType(ValType expected, Value* value); + [[nodiscard]] bool popWithType(ResultType expected, ValueVector* values); + template + [[nodiscard]] bool popWithTypes(ValTypeSpanT expected, ValueVector* values); + [[nodiscard]] bool popWithRefType(Value* value, StackType* type); + // Check that the top of the value stack has type `expected`, bearing in + // mind that it may be a block type, hence involving multiple values. + // + // If the block's stack contains polymorphic values at its base (because we + // are in unreachable code) then suitable extra values are inserted into the + // value stack, as controlled by `rewriteStackTypes`: if this is true, + // polymorphic values have their types created/updated from `expected`. If + // it is false, such values are left as `StackType::bottom()`. + // + // If `values` is non-null, it is filled in with Value components of the + // relevant stack entries, including those of any new entries created. + [[nodiscard]] bool checkTopTypeMatches(ResultType expected, + ValueVector* values, + bool rewriteStackTypes); + + [[nodiscard]] bool pushControl(LabelKind kind, BlockType type); + [[nodiscard]] bool checkStackAtEndOfBlock(ResultType* type, + ValueVector* values); + [[nodiscard]] bool getControl(uint32_t relativeDepth, Control** controlEntry); + [[nodiscard]] bool checkBranchValueAndPush(uint32_t relativeDepth, + ResultType* type, + ValueVector* values); + [[nodiscard]] bool checkBrTableEntryAndPush(uint32_t* relativeDepth, + ResultType prevBranchType, + ResultType* branchType, + ValueVector* branchValues); + + [[nodiscard]] bool push(StackType t) { return valueStack_.emplaceBack(t); } + [[nodiscard]] bool push(ValType t) { return valueStack_.emplaceBack(t); } + [[nodiscard]] bool push(TypeAndValue tv) { return valueStack_.append(tv); } + [[nodiscard]] bool push(ResultType t) { + for (size_t i = 0; i < t.length(); i++) { + if (!push(t[i])) { + return false; + } + } + return true; + } + void infalliblePush(StackType t) { valueStack_.infallibleEmplaceBack(t); } + void infalliblePush(ValType t) { + valueStack_.infallibleEmplaceBack(StackType(t)); + } + void infalliblePush(TypeAndValue tv) { valueStack_.infallibleAppend(tv); } + + void afterUnconditionalBranch() { + valueStack_.shrinkTo(controlStack_.back().valueStackBase()); + controlStack_.back().setPolymorphicBase(); + } + + inline bool checkIsSubtypeOf(FieldType actual, FieldType expected); + + inline bool checkIsSubtypeOf(RefType actual, RefType expected) { + return checkIsSubtypeOf(ValType(actual).fieldType(), + ValType(expected).fieldType()); + } + inline bool checkIsSubtypeOf(ValType actual, ValType expected) { + return checkIsSubtypeOf(actual.fieldType(), expected.fieldType()); + } + + inline bool checkIsSubtypeOf(ResultType params, ResultType results); + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + inline bool checkIsSubtypeOf(uint32_t actualTypeIndex, + uint32_t expectedTypeIndex); +#endif + + public: +#ifdef DEBUG + explicit OpIter(const ModuleEnvironment& env, Decoder& decoder, + Kind kind = OpIter::Func) + : kind_(kind), + d_(decoder), + env_(env), + maxInitializedGlobalsIndexPlus1_(0), + op_(OpBytes(Op::Limit)), + offsetOfLastReadOp_(0) {} +#else + explicit OpIter(const ModuleEnvironment& env, Decoder& decoder, + Kind kind = OpIter::Func) + : kind_(kind), + d_(decoder), + env_(env), + maxInitializedGlobalsIndexPlus1_(0), + offsetOfLastReadOp_(0) {} +#endif + + // Return the decoding byte offset. + uint32_t currentOffset() const { return d_.currentOffset(); } + + // Return the offset within the entire module of the last-read op. + size_t lastOpcodeOffset() const { + return offsetOfLastReadOp_ ? offsetOfLastReadOp_ : d_.currentOffset(); + } + + // Return a BytecodeOffset describing where the current op should be reported + // to trap/call. + BytecodeOffset bytecodeOffset() const { + return BytecodeOffset(lastOpcodeOffset()); + } + + // Test whether the iterator has reached the end of the buffer. + bool done() const { return d_.done(); } + + // Return a pointer to the end of the buffer being decoded by this iterator. + const uint8_t* end() const { return d_.end(); } + + // Report a general failure. + [[nodiscard]] bool fail(const char* msg) MOZ_COLD; + + // Report a general failure with a context + [[nodiscard]] bool fail_ctx(const char* fmt, const char* context) MOZ_COLD; + + // Report an unrecognized opcode. + [[nodiscard]] bool unrecognizedOpcode(const OpBytes* expr) MOZ_COLD; + + // Return whether the innermost block has a polymorphic base of its stack. + // Ideally this accessor would be removed; consider using something else. + bool currentBlockHasPolymorphicBase() const { + return !controlStack_.empty() && controlStack_.back().polymorphicBase(); + } + + // ------------------------------------------------------------------------ + // Decoding and validation interface. + + // Initialization and termination + + [[nodiscard]] bool startFunction(uint32_t funcIndex, + const ValTypeVector& locals); + [[nodiscard]] bool endFunction(const uint8_t* bodyEnd); + + [[nodiscard]] bool startInitExpr(ValType expected, + uint32_t maxInitializedGlobalsIndexPlus1); + [[nodiscard]] bool endInitExpr(); + + // Value and reference types + + [[nodiscard]] bool readValType(ValType* type); + [[nodiscard]] bool readHeapType(bool nullable, RefType* type); + + // Instructions + + [[nodiscard]] bool readOp(OpBytes* op); + [[nodiscard]] bool readReturn(ValueVector* values); + [[nodiscard]] bool readBlock(ResultType* paramType); + [[nodiscard]] bool readLoop(ResultType* paramType); + [[nodiscard]] bool readIf(ResultType* paramType, Value* condition); + [[nodiscard]] bool readElse(ResultType* paramType, ResultType* resultType, + ValueVector* thenResults); + [[nodiscard]] bool readEnd(LabelKind* kind, ResultType* type, + ValueVector* results, + ValueVector* resultsForEmptyElse); + void popEnd(); + [[nodiscard]] bool readBr(uint32_t* relativeDepth, ResultType* type, + ValueVector* values); + [[nodiscard]] bool readBrIf(uint32_t* relativeDepth, ResultType* type, + ValueVector* values, Value* condition); + [[nodiscard]] bool readBrTable(Uint32Vector* depths, uint32_t* defaultDepth, + ResultType* defaultBranchType, + ValueVector* branchValues, Value* index); + [[nodiscard]] bool readTry(ResultType* type); + [[nodiscard]] bool readCatch(LabelKind* kind, uint32_t* tagIndex, + ResultType* paramType, ResultType* resultType, + ValueVector* tryResults); + [[nodiscard]] bool readCatchAll(LabelKind* kind, ResultType* paramType, + ResultType* resultType, + ValueVector* tryResults); + [[nodiscard]] bool readDelegate(uint32_t* relativeDepth, + ResultType* resultType, + ValueVector* tryResults); + void popDelegate(); + [[nodiscard]] bool readThrow(uint32_t* tagIndex, ValueVector* argValues); + [[nodiscard]] bool readRethrow(uint32_t* relativeDepth); + [[nodiscard]] bool readUnreachable(); + [[nodiscard]] bool readDrop(); + [[nodiscard]] bool readUnary(ValType operandType, Value* input); + [[nodiscard]] bool readConversion(ValType operandType, ValType resultType, + Value* input); + [[nodiscard]] bool readBinary(ValType operandType, Value* lhs, Value* rhs); + [[nodiscard]] bool readComparison(ValType operandType, Value* lhs, + Value* rhs); + [[nodiscard]] bool readTernary(ValType operandType, Value* v0, Value* v1, + Value* v2); + [[nodiscard]] bool readLoad(ValType resultType, uint32_t byteSize, + LinearMemoryAddress* addr); + [[nodiscard]] bool readStore(ValType resultType, uint32_t byteSize, + LinearMemoryAddress* addr, Value* value); + [[nodiscard]] bool readTeeStore(ValType resultType, uint32_t byteSize, + LinearMemoryAddress* addr, + Value* value); + [[nodiscard]] bool readNop(); + [[nodiscard]] bool readMemorySize(); + [[nodiscard]] bool readMemoryGrow(Value* input); + [[nodiscard]] bool readSelect(bool typed, StackType* type, Value* trueValue, + Value* falseValue, Value* condition); + [[nodiscard]] bool readGetLocal(const ValTypeVector& locals, uint32_t* id); + [[nodiscard]] bool readSetLocal(const ValTypeVector& locals, uint32_t* id, + Value* value); + [[nodiscard]] bool readTeeLocal(const ValTypeVector& locals, uint32_t* id, + Value* value); + [[nodiscard]] bool readGetGlobal(uint32_t* id); + [[nodiscard]] bool readSetGlobal(uint32_t* id, Value* value); + [[nodiscard]] bool readTeeGlobal(uint32_t* id, Value* value); + [[nodiscard]] bool readI32Const(int32_t* i32); + [[nodiscard]] bool readI64Const(int64_t* i64); + [[nodiscard]] bool readF32Const(float* f32); + [[nodiscard]] bool readF64Const(double* f64); + [[nodiscard]] bool readRefFunc(uint32_t* funcIndex); + [[nodiscard]] bool readRefNull(RefType* type); + [[nodiscard]] bool readRefIsNull(Value* input); + [[nodiscard]] bool readRefAsNonNull(Value* input); + [[nodiscard]] bool readBrOnNull(uint32_t* relativeDepth, ResultType* type, + ValueVector* values, Value* condition); + [[nodiscard]] bool readBrOnNonNull(uint32_t* relativeDepth, ResultType* type, + ValueVector* values, Value* condition); + [[nodiscard]] bool readCall(uint32_t* funcTypeIndex, ValueVector* argValues); + [[nodiscard]] bool readCallIndirect(uint32_t* funcTypeIndex, + uint32_t* tableIndex, Value* callee, + ValueVector* argValues); +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + [[nodiscard]] bool readCallRef(const FuncType** funcType, Value* callee, + ValueVector* argValues); +#endif + [[nodiscard]] bool readOldCallDirect(uint32_t numFuncImports, + uint32_t* funcTypeIndex, + ValueVector* argValues); + [[nodiscard]] bool readOldCallIndirect(uint32_t* funcTypeIndex, Value* callee, + ValueVector* argValues); + [[nodiscard]] bool readWake(LinearMemoryAddress* addr, Value* count); + [[nodiscard]] bool readWait(LinearMemoryAddress* addr, + ValType valueType, uint32_t byteSize, + Value* value, Value* timeout); + [[nodiscard]] bool readFence(); + [[nodiscard]] bool readAtomicLoad(LinearMemoryAddress* addr, + ValType resultType, uint32_t byteSize); + [[nodiscard]] bool readAtomicStore(LinearMemoryAddress* addr, + ValType resultType, uint32_t byteSize, + Value* value); + [[nodiscard]] bool readAtomicRMW(LinearMemoryAddress* addr, + ValType resultType, uint32_t byteSize, + Value* value); + [[nodiscard]] bool readAtomicCmpXchg(LinearMemoryAddress* addr, + ValType resultType, uint32_t byteSize, + Value* oldValue, Value* newValue); + [[nodiscard]] bool readMemOrTableCopy(bool isMem, + uint32_t* dstMemOrTableIndex, + Value* dst, + uint32_t* srcMemOrTableIndex, + Value* src, Value* len); + [[nodiscard]] bool readDataOrElemDrop(bool isData, uint32_t* segIndex); + [[nodiscard]] bool readMemFill(Value* start, Value* val, Value* len); + [[nodiscard]] bool readMemOrTableInit(bool isMem, uint32_t* segIndex, + uint32_t* dstTableIndex, Value* dst, + Value* src, Value* len); + [[nodiscard]] bool readTableFill(uint32_t* tableIndex, Value* start, + Value* val, Value* len); + [[nodiscard]] bool readMemDiscard(Value* start, Value* len); + [[nodiscard]] bool readTableGet(uint32_t* tableIndex, Value* index); + [[nodiscard]] bool readTableGrow(uint32_t* tableIndex, Value* initValue, + Value* delta); + [[nodiscard]] bool readTableSet(uint32_t* tableIndex, Value* index, + Value* value); + + [[nodiscard]] bool readTableSize(uint32_t* tableIndex); + +#ifdef ENABLE_WASM_GC + [[nodiscard]] bool readStructNew(uint32_t* typeIndex, ValueVector* argValues); + [[nodiscard]] bool readStructNewDefault(uint32_t* typeIndex); + [[nodiscard]] bool readStructGet(uint32_t* typeIndex, uint32_t* fieldIndex, + FieldWideningOp wideningOp, Value* ptr); + [[nodiscard]] bool readStructSet(uint32_t* typeIndex, uint32_t* fieldIndex, + Value* ptr, Value* val); + [[nodiscard]] bool readArrayNew(uint32_t* typeIndex, Value* numElements, + Value* argValue); + [[nodiscard]] bool readArrayNewFixed(uint32_t* typeIndex, + uint32_t* numElements, + ValueVector* values); + [[nodiscard]] bool readArrayNewDefault(uint32_t* typeIndex, + Value* numElements); + [[nodiscard]] bool readArrayNewData(uint32_t* typeIndex, uint32_t* segIndex, + Value* offset, Value* numElements); + [[nodiscard]] bool readArrayNewElem(uint32_t* typeIndex, uint32_t* segIndex, + Value* offset, Value* numElements); + [[nodiscard]] bool readArrayGet(uint32_t* typeIndex, + FieldWideningOp wideningOp, Value* index, + Value* ptr); + [[nodiscard]] bool readArraySet(uint32_t* typeIndex, Value* val, Value* index, + Value* ptr); + [[nodiscard]] bool readArrayLen(bool decodeIgnoredTypeIndex, Value* ptr); + [[nodiscard]] bool readArrayCopy(int32_t* elemSize, bool* elemsAreRefTyped, + Value* dstArray, Value* dstIndex, + Value* srcArray, Value* srcIndex, + Value* numElements); + [[nodiscard]] bool readRefTestV5(RefType* sourceType, uint32_t* typeIndex, + Value* ref); + [[nodiscard]] bool readRefCastV5(RefType* sourceType, uint32_t* typeIndex, + Value* ref); + [[nodiscard]] bool readBrOnCastV5(uint32_t* labelRelativeDepth, + RefType* sourceType, + uint32_t* castTypeIndex, + ResultType* labelType, ValueVector* values); + [[nodiscard]] bool readBrOnCastHeapV5(bool nullable, + uint32_t* labelRelativeDepth, + RefType* sourceType, RefType* destType, + ResultType* labelType, + ValueVector* values); + [[nodiscard]] bool readBrOnCastFailV5(uint32_t* labelRelativeDepth, + RefType* sourceType, + uint32_t* castTypeIndex, + ResultType* labelType, + ValueVector* values); + [[nodiscard]] bool readBrOnCastFailHeapV5( + bool nullable, uint32_t* labelRelativeDepth, RefType* sourceType, + RefType* destType, ResultType* labelType, ValueVector* values); + [[nodiscard]] bool readRefTest(bool nullable, RefType* sourceType, + RefType* destType, Value* ref); + [[nodiscard]] bool readRefCast(bool nullable, RefType* sourceType, + RefType* destType, Value* ref); + [[nodiscard]] bool readBrOnCast(bool* onSuccess, uint32_t* labelRelativeDepth, + RefType* sourceType, RefType* destType, + ResultType* labelType, ValueVector* values); + [[nodiscard]] bool checkBrOnCastCommonV5(uint32_t labelRelativeDepth, + RefType* sourceType, + ValType castToType, + ResultType* labelType, + ValueVector* values); + [[nodiscard]] bool checkBrOnCastFailCommonV5(uint32_t labelRelativeDepth, + RefType* sourceType, + ValType castToType, + ResultType* labelType, + ValueVector* values); + [[nodiscard]] bool readBrOnNonStructV5(uint32_t* labelRelativeDepth, + ResultType* labelType, + ValueVector* values); + [[nodiscard]] bool readRefConversion(RefType operandType, RefType resultType, + Value* operandValue); +#endif + +#ifdef ENABLE_WASM_SIMD + [[nodiscard]] bool readLaneIndex(uint32_t inputLanes, uint32_t* laneIndex); + [[nodiscard]] bool readExtractLane(ValType resultType, uint32_t inputLanes, + uint32_t* laneIndex, Value* input); + [[nodiscard]] bool readReplaceLane(ValType operandType, uint32_t inputLanes, + uint32_t* laneIndex, Value* baseValue, + Value* operand); + [[nodiscard]] bool readVectorShift(Value* baseValue, Value* shift); + [[nodiscard]] bool readVectorShuffle(Value* v1, Value* v2, V128* selectMask); + [[nodiscard]] bool readV128Const(V128* value); + [[nodiscard]] bool readLoadSplat(uint32_t byteSize, + LinearMemoryAddress* addr); + [[nodiscard]] bool readLoadExtend(LinearMemoryAddress* addr); + [[nodiscard]] bool readLoadLane(uint32_t byteSize, + LinearMemoryAddress* addr, + uint32_t* laneIndex, Value* input); + [[nodiscard]] bool readStoreLane(uint32_t byteSize, + LinearMemoryAddress* addr, + uint32_t* laneIndex, Value* input); +#endif + + [[nodiscard]] bool readIntrinsic(const Intrinsic** intrinsic, + ValueVector* params); + + // At a location where readOp is allowed, peek at the next opcode + // without consuming it or updating any internal state. + // Never fails: returns uint16_t(Op::Limit) in op->b0 if it can't read. + void peekOp(OpBytes* op); + + // ------------------------------------------------------------------------ + // Stack management. + + // Set the top N result values. + void setResults(size_t count, const ValueVector& values) { + MOZ_ASSERT(valueStack_.length() >= count); + size_t base = valueStack_.length() - count; + for (size_t i = 0; i < count; i++) { + valueStack_[base + i].setValue(values[i]); + } + } + + bool getResults(size_t count, ValueVector* values) { + MOZ_ASSERT(valueStack_.length() >= count); + if (!values->resize(count)) { + return false; + } + size_t base = valueStack_.length() - count; + for (size_t i = 0; i < count; i++) { + (*values)[i] = valueStack_[base + i].value(); + } + return true; + } + + // Set the result value of the current top-of-value-stack expression. + void setResult(Value value) { valueStack_.back().setValue(value); } + + // Return the result value of the current top-of-value-stack expression. + Value getResult() { return valueStack_.back().value(); } + + // Return a reference to the top of the control stack. + ControlItem& controlItem() { return controlStack_.back().controlItem(); } + + // Return a reference to an element in the control stack. + ControlItem& controlItem(uint32_t relativeDepth) { + return controlStack_[controlStack_.length() - 1 - relativeDepth] + .controlItem(); + } + + // Return the LabelKind of an element in the control stack. + LabelKind controlKind(uint32_t relativeDepth) { + return controlStack_[controlStack_.length() - 1 - relativeDepth].kind(); + } + + // Return a reference to the outermost element on the control stack. + ControlItem& controlOutermost() { return controlStack_[0].controlItem(); } + + // Test whether the control-stack is empty, meaning we've consumed the final + // end of the function body. + bool controlStackEmpty() const { return controlStack_.empty(); } + + // Return the depth of the control stack. + size_t controlStackDepth() const { return controlStack_.length(); } + + // Find the innermost control item of a specific kind, starting to search from + // a certain relative depth, and returning true if such innermost control item + // is found. The relative depth of the found item is returned via a parameter. + bool controlFindInnermostFrom(LabelKind kind, uint32_t fromRelativeDepth, + uint32_t* foundRelativeDepth) { + int32_t fromAbsoluteDepth = controlStack_.length() - fromRelativeDepth - 1; + for (int32_t i = fromAbsoluteDepth; i >= 0; i--) { + if (controlStack_[i].kind() == kind) { + *foundRelativeDepth = controlStack_.length() - 1 - i; + return true; + } + } + return false; + } + + bool controlFindInnermost(LabelKind kind, uint32_t* foundRelativeDepth) { + return controlFindInnermostFrom(kind, 0, foundRelativeDepth); + } +}; + +template +inline bool OpIter::checkIsSubtypeOf(FieldType subType, + FieldType superType) { + return CheckIsSubtypeOf(d_, env_, lastOpcodeOffset(), subType, superType); +} + +template +inline bool OpIter::checkIsSubtypeOf(ResultType params, + ResultType results) { + if (params.length() != results.length()) { + UniqueChars error( + JS_smprintf("type mismatch: expected %zu values, got %zu values", + results.length(), params.length())); + if (!error) { + return false; + } + return fail(error.get()); + } + for (uint32_t i = 0; i < params.length(); i++) { + ValType param = params[i]; + ValType result = results[i]; + if (!checkIsSubtypeOf(param, result)) { + return false; + } + } + return true; +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +template +inline bool OpIter::checkIsSubtypeOf(uint32_t actualTypeIndex, + uint32_t expectedTypeIndex) { + const TypeDef& actualTypeDef = env_.types->type(actualTypeIndex); + const TypeDef& expectedTypeDef = env_.types->type(expectedTypeIndex); + return CheckIsSubtypeOf( + d_, env_, lastOpcodeOffset(), + ValType(RefType::fromTypeDef(&actualTypeDef, true)), + ValType(RefType::fromTypeDef(&expectedTypeDef, true))); +} +#endif + +template +inline bool OpIter::unrecognizedOpcode(const OpBytes* expr) { + UniqueChars error(JS_smprintf("unrecognized opcode: %x %x", expr->b0, + IsPrefixByte(expr->b0) ? expr->b1 : 0)); + if (!error) { + return false; + } + + return fail(error.get()); +} + +template +inline bool OpIter::fail(const char* msg) { + return d_.fail(lastOpcodeOffset(), msg); +} + +template +inline bool OpIter::fail_ctx(const char* fmt, const char* context) { + UniqueChars error(JS_smprintf(fmt, context)); + if (!error) { + return false; + } + return fail(error.get()); +} + +template +inline bool OpIter::failEmptyStack() { + return valueStack_.empty() ? fail("popping value from empty stack") + : fail("popping value from outside block"); +} + +// This function pops exactly one value from the stack, yielding Bottom types in +// various cases and therefore making it the caller's responsibility to do the +// right thing for StackType::Bottom. Prefer (pop|top)WithType. This is an +// optimization for the super-common case where the caller is statically +// expecting the resulttype `[valtype]`. +template +inline bool OpIter::popStackType(StackType* type, Value* value) { + Control& block = controlStack_.back(); + + MOZ_ASSERT(valueStack_.length() >= block.valueStackBase()); + if (MOZ_UNLIKELY(valueStack_.length() == block.valueStackBase())) { + // If the base of this block's stack is polymorphic, then we can pop a + // dummy value of the bottom type; it won't be used since we're in + // unreachable code. + if (block.polymorphicBase()) { + *type = StackType::bottom(); + *value = Value(); + + // Maintain the invariant that, after a pop, there is always memory + // reserved to push a value infallibly. + return valueStack_.reserve(valueStack_.length() + 1); + } + + return failEmptyStack(); + } + + TypeAndValue& tv = valueStack_.back(); + *type = tv.type(); + *value = tv.value(); + valueStack_.popBack(); + return true; +} + +// This function pops exactly one value from the stack, checking that it has the +// expected type which can either be a specific value type or the bottom type. +template +inline bool OpIter::popWithType(ValType expectedType, Value* value, + StackType* stackType) { + if (!popStackType(stackType, value)) { + return false; + } + + return stackType->isStackBottom() || + checkIsSubtypeOf(stackType->valType(), expectedType); +} + +// This function pops exactly one value from the stack, checking that it has the +// expected type which can either be a specific value type or the bottom type. +template +inline bool OpIter::popWithType(ValType expectedType, Value* value) { + StackType stackType; + return popWithType(expectedType, value, &stackType); +} + +template +inline bool OpIter::popWithType(ResultType expected, + ValueVector* values) { + return popWithTypes(expected, values); +} + +// Pops each of the given expected types (in reverse, because it's a stack). +template +template +inline bool OpIter::popWithTypes(ValTypeSpanT expected, + ValueVector* values) { + size_t expectedLength = expected.size(); + if (!values->resize(expectedLength)) { + return false; + } + for (size_t i = 0; i < expectedLength; i++) { + size_t reverseIndex = expectedLength - i - 1; + ValType expectedType = expected[reverseIndex]; + Value* value = &(*values)[reverseIndex]; + if (!popWithType(expectedType, value)) { + return false; + } + } + return true; +} + +// This function pops exactly one value from the stack, checking that it is a +// reference type. +template +inline bool OpIter::popWithRefType(Value* value, StackType* type) { + if (!popStackType(type, value)) { + return false; + } + + if (type->isStackBottom() || type->valType().isRefType()) { + return true; + } + + UniqueChars actualText = ToString(type->valType(), env_.types); + if (!actualText) { + return false; + } + + UniqueChars error(JS_smprintf( + "type mismatch: expression has type %s but expected a reference type", + actualText.get())); + if (!error) { + return false; + } + + return fail(error.get()); +} + +template +inline bool OpIter::checkTopTypeMatches(ResultType expected, + ValueVector* values, + bool rewriteStackTypes) { + if (expected.empty()) { + return true; + } + + Control& block = controlStack_.back(); + + size_t expectedLength = expected.length(); + if (values && !values->resize(expectedLength)) { + return false; + } + + for (size_t i = 0; i != expectedLength; i++) { + // We're iterating as-if we were popping each expected/actual type one by + // one, which means iterating the array of expected results backwards. + // The "current" value stack length refers to what the value stack length + // would have been if we were popping it. + size_t reverseIndex = expectedLength - i - 1; + ValType expectedType = expected[reverseIndex]; + auto collectValue = [&](const Value& v) { + if (values) { + (*values)[reverseIndex] = v; + } + }; + + size_t currentValueStackLength = valueStack_.length() - i; + + MOZ_ASSERT(currentValueStackLength >= block.valueStackBase()); + if (currentValueStackLength == block.valueStackBase()) { + if (!block.polymorphicBase()) { + return failEmptyStack(); + } + + // If the base of this block's stack is polymorphic, then we can just + // pull out as many fake values as we need to validate, and create dummy + // stack entries accordingly; they won't be used since we're in + // unreachable code. However, if `rewriteStackTypes` is true, we must + // set the types on these new entries to whatever `expected` requires + // them to be. + TypeAndValue newTandV = + rewriteStackTypes ? TypeAndValue(expectedType) : TypeAndValue(); + if (!valueStack_.insert(valueStack_.begin() + currentValueStackLength, + newTandV)) { + return false; + } + + collectValue(Value()); + } else { + TypeAndValue& observed = valueStack_[currentValueStackLength - 1]; + + if (observed.type().isStackBottom()) { + collectValue(Value()); + } else { + if (!checkIsSubtypeOf(observed.type().valType(), expectedType)) { + return false; + } + + collectValue(observed.value()); + } + + if (rewriteStackTypes) { + observed.setType(StackType(expectedType)); + } + } + } + return true; +} + +template +inline bool OpIter::pushControl(LabelKind kind, BlockType type) { + ResultType paramType = type.params(); + + ValueVector values; + if (!checkTopTypeMatches(paramType, &values, /*rewriteStackTypes=*/true)) { + return false; + } + MOZ_ASSERT(valueStack_.length() >= paramType.length()); + uint32_t valueStackBase = valueStack_.length() - paramType.length(); + return controlStack_.emplaceBack(kind, type, valueStackBase); +} + +template +inline bool OpIter::checkStackAtEndOfBlock(ResultType* expectedType, + ValueVector* values) { + Control& block = controlStack_.back(); + *expectedType = block.type().results(); + + MOZ_ASSERT(valueStack_.length() >= block.valueStackBase()); + if (expectedType->length() < valueStack_.length() - block.valueStackBase()) { + return fail("unused values not explicitly dropped by end of block"); + } + + return checkTopTypeMatches(*expectedType, values, + /*rewriteStackTypes=*/true); +} + +template +inline bool OpIter::getControl(uint32_t relativeDepth, + Control** controlEntry) { + if (relativeDepth >= controlStack_.length()) { + return fail("branch depth exceeds current nesting level"); + } + + *controlEntry = &controlStack_[controlStack_.length() - 1 - relativeDepth]; + return true; +} + +template +inline bool OpIter::readBlockType(BlockType* type) { + uint8_t nextByte; + if (!d_.peekByte(&nextByte)) { + return fail("unable to read block type"); + } + + if (nextByte == uint8_t(TypeCode::BlockVoid)) { + d_.uncheckedReadFixedU8(); + *type = BlockType::VoidToVoid(); + return true; + } + + if ((nextByte & SLEB128SignMask) == SLEB128SignBit) { + ValType v; + if (!readValType(&v)) { + return false; + } + *type = BlockType::VoidToSingle(v); + return true; + } + + int32_t x; + if (!d_.readVarS32(&x) || x < 0 || uint32_t(x) >= env_.types->length()) { + return fail("invalid block type type index"); + } + + const TypeDef* typeDef = &env_.types->type(x); + if (!typeDef->isFuncType()) { + return fail("block type type index must be func type"); + } + + *type = BlockType::Func(typeDef->funcType()); + + return true; +} + +template +inline bool OpIter::readOp(OpBytes* op) { + MOZ_ASSERT(!controlStack_.empty()); + + offsetOfLastReadOp_ = d_.currentOffset(); + + if (MOZ_UNLIKELY(!d_.readOp(op))) { + return fail("unable to read opcode"); + } + +#ifdef DEBUG + op_ = *op; +#endif + + return true; +} + +template +inline void OpIter::peekOp(OpBytes* op) { + const uint8_t* pos = d_.currentPosition(); + + if (MOZ_UNLIKELY(!d_.readOp(op))) { + op->b0 = uint16_t(Op::Limit); + } + + d_.rollbackPosition(pos); +} + +template +inline bool OpIter::startFunction(uint32_t funcIndex, + const ValTypeVector& locals) { + MOZ_ASSERT(kind_ == OpIter::Func); + MOZ_ASSERT(elseParamStack_.empty()); + MOZ_ASSERT(valueStack_.empty()); + MOZ_ASSERT(controlStack_.empty()); + MOZ_ASSERT(op_.b0 == uint16_t(Op::Limit)); + MOZ_ASSERT(maxInitializedGlobalsIndexPlus1_ == 0); + BlockType type = BlockType::FuncResults(*env_.funcs[funcIndex].type); + + size_t numArgs = env_.funcs[funcIndex].type->args().length(); + if (!unsetLocals_.init(locals, numArgs)) { + return false; + } + + return pushControl(LabelKind::Body, type); +} + +template +inline bool OpIter::endFunction(const uint8_t* bodyEnd) { + if (d_.currentPosition() != bodyEnd) { + return fail("function body length mismatch"); + } + + if (!controlStack_.empty()) { + return fail("unbalanced function body control flow"); + } + MOZ_ASSERT(elseParamStack_.empty()); + MOZ_ASSERT(unsetLocals_.empty()); + +#ifdef DEBUG + op_ = OpBytes(Op::Limit); +#endif + valueStack_.clear(); + return true; +} + +template +inline bool OpIter::startInitExpr( + ValType expected, uint32_t maxInitializedGlobalsIndexPlus1) { + MOZ_ASSERT(kind_ == OpIter::InitExpr); + MOZ_ASSERT(elseParamStack_.empty()); + MOZ_ASSERT(valueStack_.empty()); + MOZ_ASSERT(controlStack_.empty()); + MOZ_ASSERT(maxInitializedGlobalsIndexPlus1_ == 0); + MOZ_ASSERT(op_.b0 == uint16_t(Op::Limit)); + + // GC allows accessing any previously defined global, not just those that are + // imported and immutable. + if (env_.features.gc) { + maxInitializedGlobalsIndexPlus1_ = maxInitializedGlobalsIndexPlus1; + } + + BlockType type = BlockType::VoidToSingle(expected); + return pushControl(LabelKind::Body, type); +} + +template +inline bool OpIter::endInitExpr() { + MOZ_ASSERT(controlStack_.empty()); + MOZ_ASSERT(elseParamStack_.empty()); + +#ifdef DEBUG + op_ = OpBytes(Op::Limit); +#endif + valueStack_.clear(); + return true; +} + +template +inline bool OpIter::readValType(ValType* type) { + return d_.readValType(*env_.types, env_.features, type); +} + +template +inline bool OpIter::readHeapType(bool nullable, RefType* type) { + return d_.readHeapType(*env_.types, env_.features, nullable, type); +} + +template +inline bool OpIter::readReturn(ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::Return); + + Control& body = controlStack_[0]; + MOZ_ASSERT(body.kind() == LabelKind::Body); + + if (!popWithType(body.resultType(), values)) { + return false; + } + + afterUnconditionalBranch(); + return true; +} + +template +inline bool OpIter::readBlock(ResultType* paramType) { + MOZ_ASSERT(Classify(op_) == OpKind::Block); + + BlockType type; + if (!readBlockType(&type)) { + return false; + } + + *paramType = type.params(); + return pushControl(LabelKind::Block, type); +} + +template +inline bool OpIter::readLoop(ResultType* paramType) { + MOZ_ASSERT(Classify(op_) == OpKind::Loop); + + BlockType type; + if (!readBlockType(&type)) { + return false; + } + + *paramType = type.params(); + return pushControl(LabelKind::Loop, type); +} + +template +inline bool OpIter::readIf(ResultType* paramType, Value* condition) { + MOZ_ASSERT(Classify(op_) == OpKind::If); + + BlockType type; + if (!readBlockType(&type)) { + return false; + } + + if (!popWithType(ValType::I32, condition)) { + return false; + } + + if (!pushControl(LabelKind::Then, type)) { + return false; + } + + *paramType = type.params(); + size_t paramsLength = type.params().length(); + return elseParamStack_.append(valueStack_.end() - paramsLength, paramsLength); +} + +template +inline bool OpIter::readElse(ResultType* paramType, + ResultType* resultType, + ValueVector* thenResults) { + MOZ_ASSERT(Classify(op_) == OpKind::Else); + + Control& block = controlStack_.back(); + if (block.kind() != LabelKind::Then) { + return fail("else can only be used within an if"); + } + + *paramType = block.type().params(); + if (!checkStackAtEndOfBlock(resultType, thenResults)) { + return false; + } + + valueStack_.shrinkTo(block.valueStackBase()); + + size_t nparams = block.type().params().length(); + MOZ_ASSERT(elseParamStack_.length() >= nparams); + valueStack_.infallibleAppend(elseParamStack_.end() - nparams, nparams); + elseParamStack_.shrinkBy(nparams); + + // Reset local state to the beginning of the 'if' block for the new block + // started by 'else'. + unsetLocals_.resetToBlock(controlStack_.length() - 1); + + block.switchToElse(); + return true; +} + +template +inline bool OpIter::readEnd(LabelKind* kind, ResultType* type, + ValueVector* results, + ValueVector* resultsForEmptyElse) { + MOZ_ASSERT(Classify(op_) == OpKind::End); + + Control& block = controlStack_.back(); + + if (!checkStackAtEndOfBlock(type, results)) { + return false; + } + + if (block.kind() == LabelKind::Then) { + ResultType params = block.type().params(); + // If an `if` block ends with `end` instead of `else`, then the `else` block + // implicitly passes the `if` parameters as the `else` results. In that + // case, assert that the `if`'s param type matches the result type. + if (params != block.type().results()) { + return fail("if without else with a result value"); + } + + size_t nparams = params.length(); + MOZ_ASSERT(elseParamStack_.length() >= nparams); + if (!resultsForEmptyElse->resize(nparams)) { + return false; + } + const TypeAndValue* elseParams = elseParamStack_.end() - nparams; + for (size_t i = 0; i < nparams; i++) { + (*resultsForEmptyElse)[i] = elseParams[i].value(); + } + elseParamStack_.shrinkBy(nparams); + } + + *kind = block.kind(); + return true; +} + +template +inline void OpIter::popEnd() { + MOZ_ASSERT(Classify(op_) == OpKind::End); + + controlStack_.popBack(); + unsetLocals_.resetToBlock(controlStack_.length()); +} + +template +inline bool OpIter::checkBranchValueAndPush(uint32_t relativeDepth, + ResultType* type, + ValueVector* values) { + Control* block = nullptr; + if (!getControl(relativeDepth, &block)) { + return false; + } + + *type = block->branchTargetType(); + return checkTopTypeMatches(*type, values, /*rewriteStackTypes=*/false); +} + +template +inline bool OpIter::readBr(uint32_t* relativeDepth, ResultType* type, + ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::Br); + + if (!readVarU32(relativeDepth)) { + return fail("unable to read br depth"); + } + + if (!checkBranchValueAndPush(*relativeDepth, type, values)) { + return false; + } + + afterUnconditionalBranch(); + return true; +} + +template +inline bool OpIter::readBrIf(uint32_t* relativeDepth, ResultType* type, + ValueVector* values, Value* condition) { + MOZ_ASSERT(Classify(op_) == OpKind::BrIf); + + if (!readVarU32(relativeDepth)) { + return fail("unable to read br_if depth"); + } + + if (!popWithType(ValType::I32, condition)) { + return false; + } + + return checkBranchValueAndPush(*relativeDepth, type, values); +} + +#define UNKNOWN_ARITY UINT32_MAX + +template +inline bool OpIter::checkBrTableEntryAndPush( + uint32_t* relativeDepth, ResultType prevBranchType, ResultType* type, + ValueVector* branchValues) { + if (!readVarU32(relativeDepth)) { + return fail("unable to read br_table depth"); + } + + Control* block = nullptr; + if (!getControl(*relativeDepth, &block)) { + return false; + } + + *type = block->branchTargetType(); + + if (prevBranchType.valid()) { + if (prevBranchType.length() != type->length()) { + return fail("br_table targets must all have the same arity"); + } + + // Avoid re-collecting the same values for subsequent branch targets. + branchValues = nullptr; + } + + return checkTopTypeMatches(*type, branchValues, /*rewriteStackTypes=*/false); +} + +template +inline bool OpIter::readBrTable(Uint32Vector* depths, + uint32_t* defaultDepth, + ResultType* defaultBranchType, + ValueVector* branchValues, + Value* index) { + MOZ_ASSERT(Classify(op_) == OpKind::BrTable); + + uint32_t tableLength; + if (!readVarU32(&tableLength)) { + return fail("unable to read br_table table length"); + } + + if (tableLength > MaxBrTableElems) { + return fail("br_table too big"); + } + + if (!popWithType(ValType::I32, index)) { + return false; + } + + if (!depths->resize(tableLength)) { + return false; + } + + ResultType prevBranchType; + for (uint32_t i = 0; i < tableLength; i++) { + ResultType branchType; + if (!checkBrTableEntryAndPush(&(*depths)[i], prevBranchType, &branchType, + branchValues)) { + return false; + } + prevBranchType = branchType; + } + + if (!checkBrTableEntryAndPush(defaultDepth, prevBranchType, defaultBranchType, + branchValues)) { + return false; + } + + MOZ_ASSERT(defaultBranchType->valid()); + + afterUnconditionalBranch(); + return true; +} + +#undef UNKNOWN_ARITY + +template +inline bool OpIter::readTry(ResultType* paramType) { + MOZ_ASSERT(Classify(op_) == OpKind::Try); + + BlockType type; + if (!readBlockType(&type)) { + return false; + } + + *paramType = type.params(); + return pushControl(LabelKind::Try, type); +} + +template +inline bool OpIter::readCatch(LabelKind* kind, uint32_t* tagIndex, + ResultType* paramType, + ResultType* resultType, + ValueVector* tryResults) { + MOZ_ASSERT(Classify(op_) == OpKind::Catch); + + if (!readVarU32(tagIndex)) { + return fail("expected tag index"); + } + if (*tagIndex >= env_.tags.length()) { + return fail("tag index out of range"); + } + + Control& block = controlStack_.back(); + if (block.kind() == LabelKind::CatchAll) { + return fail("catch cannot follow a catch_all"); + } + if (block.kind() != LabelKind::Try && block.kind() != LabelKind::Catch) { + return fail("catch can only be used within a try-catch"); + } + *kind = block.kind(); + *paramType = block.type().params(); + + if (!checkStackAtEndOfBlock(resultType, tryResults)) { + return false; + } + + valueStack_.shrinkTo(block.valueStackBase()); + block.switchToCatch(); + // Reset local state to the beginning of the 'try' block. + unsetLocals_.resetToBlock(controlStack_.length() - 1); + + return push(env_.tags[*tagIndex].type->resultType()); +} + +template +inline bool OpIter::readCatchAll(LabelKind* kind, ResultType* paramType, + ResultType* resultType, + ValueVector* tryResults) { + MOZ_ASSERT(Classify(op_) == OpKind::CatchAll); + + Control& block = controlStack_.back(); + if (block.kind() != LabelKind::Try && block.kind() != LabelKind::Catch) { + return fail("catch_all can only be used within a try-catch"); + } + *kind = block.kind(); + *paramType = block.type().params(); + + if (!checkStackAtEndOfBlock(resultType, tryResults)) { + return false; + } + + valueStack_.shrinkTo(block.valueStackBase()); + block.switchToCatchAll(); + // Reset local state to the beginning of the 'try' block. + unsetLocals_.resetToBlock(controlStack_.length() - 1); + + return true; +} + +template +inline bool OpIter::readDelegate(uint32_t* relativeDepth, + ResultType* resultType, + ValueVector* tryResults) { + MOZ_ASSERT(Classify(op_) == OpKind::Delegate); + + Control& block = controlStack_.back(); + if (block.kind() != LabelKind::Try) { + return fail("delegate can only be used within a try"); + } + + uint32_t delegateDepth; + if (!readVarU32(&delegateDepth)) { + return fail("unable to read delegate depth"); + } + + // Depths for delegate start counting in the surrounding block. + if (delegateDepth >= controlStack_.length() - 1) { + return fail("delegate depth exceeds current nesting level"); + } + *relativeDepth = delegateDepth + 1; + + // Because `delegate` acts like `end` and ends the block, we will check + // the stack here. + return checkStackAtEndOfBlock(resultType, tryResults); +} + +// We need popDelegate because readDelegate cannot pop the control stack +// itself, as its caller may need to use the control item for delegate. +template +inline void OpIter::popDelegate() { + MOZ_ASSERT(Classify(op_) == OpKind::Delegate); + + controlStack_.popBack(); +} + +template +inline bool OpIter::readThrow(uint32_t* tagIndex, + ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::Throw); + + if (!readVarU32(tagIndex)) { + return fail("expected tag index"); + } + if (*tagIndex >= env_.tags.length()) { + return fail("tag index out of range"); + } + + if (!popWithType(env_.tags[*tagIndex].type->resultType(), argValues)) { + return false; + } + + afterUnconditionalBranch(); + return true; +} + +template +inline bool OpIter::readRethrow(uint32_t* relativeDepth) { + MOZ_ASSERT(Classify(op_) == OpKind::Rethrow); + + if (!readVarU32(relativeDepth)) { + return fail("unable to read rethrow depth"); + } + + if (*relativeDepth >= controlStack_.length()) { + return fail("rethrow depth exceeds current nesting level"); + } + LabelKind kind = controlKind(*relativeDepth); + if (kind != LabelKind::Catch && kind != LabelKind::CatchAll) { + return fail("rethrow target was not a catch block"); + } + + afterUnconditionalBranch(); + return true; +} + +template +inline bool OpIter::readUnreachable() { + MOZ_ASSERT(Classify(op_) == OpKind::Unreachable); + + afterUnconditionalBranch(); + return true; +} + +template +inline bool OpIter::readDrop() { + MOZ_ASSERT(Classify(op_) == OpKind::Drop); + StackType type; + Value value; + return popStackType(&type, &value); +} + +template +inline bool OpIter::readUnary(ValType operandType, Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::Unary); + + if (!popWithType(operandType, input)) { + return false; + } + + infalliblePush(operandType); + + return true; +} + +template +inline bool OpIter::readConversion(ValType operandType, + ValType resultType, Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::Conversion); + + if (!popWithType(operandType, input)) { + return false; + } + + infalliblePush(resultType); + + return true; +} + +template +inline bool OpIter::readBinary(ValType operandType, Value* lhs, + Value* rhs) { + MOZ_ASSERT(Classify(op_) == OpKind::Binary); + + if (!popWithType(operandType, rhs)) { + return false; + } + + if (!popWithType(operandType, lhs)) { + return false; + } + + infalliblePush(operandType); + + return true; +} + +template +inline bool OpIter::readComparison(ValType operandType, Value* lhs, + Value* rhs) { + MOZ_ASSERT(Classify(op_) == OpKind::Comparison); + + if (!popWithType(operandType, rhs)) { + return false; + } + + if (!popWithType(operandType, lhs)) { + return false; + } + + infalliblePush(ValType::I32); + + return true; +} + +template +inline bool OpIter::readTernary(ValType operandType, Value* v0, + Value* v1, Value* v2) { + MOZ_ASSERT(Classify(op_) == OpKind::Ternary); + + if (!popWithType(operandType, v2)) { + return false; + } + + if (!popWithType(operandType, v1)) { + return false; + } + + if (!popWithType(operandType, v0)) { + return false; + } + + infalliblePush(operandType); + + return true; +} + +// For memories, the index is currently always a placeholder zero byte. +// +// For tables, the index is a placeholder zero byte until we get multi-table +// with the reftypes proposal. +// +// The zero-ness of the value must be checked by the caller. +template +inline bool OpIter::readMemOrTableIndex(bool isMem, uint32_t* index) { + bool readByte = isMem; + if (readByte) { + uint8_t indexTmp; + if (!readFixedU8(&indexTmp)) { + return fail("unable to read memory or table index"); + } + *index = indexTmp; + } else { + if (!readVarU32(index)) { + return fail("unable to read memory or table index"); + } + } + return true; +} + +template +inline bool OpIter::readLinearMemoryAddress( + uint32_t byteSize, LinearMemoryAddress* addr) { + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + + IndexType it = env_.memory->indexType(); + + uint32_t alignLog2; + if (!readVarU32(&alignLog2)) { + return fail("unable to read load alignment"); + } + + if (!readVarU64(&addr->offset)) { + return fail("unable to read load offset"); + } + + if (it == IndexType::I32 && addr->offset > UINT32_MAX) { + return fail("offset too large for memory type"); + } + + if (alignLog2 >= 32 || (uint32_t(1) << alignLog2) > byteSize) { + return fail("greater than natural alignment"); + } + + if (!popWithType(ToValType(it), &addr->base)) { + return false; + } + + addr->align = uint32_t(1) << alignLog2; + return true; +} + +template +inline bool OpIter::readLinearMemoryAddressAligned( + uint32_t byteSize, LinearMemoryAddress* addr) { + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + if (addr->align != byteSize) { + return fail("not natural alignment"); + } + + return true; +} + +template +inline bool OpIter::readLoad(ValType resultType, uint32_t byteSize, + LinearMemoryAddress* addr) { + MOZ_ASSERT(Classify(op_) == OpKind::Load); + + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + infalliblePush(resultType); + + return true; +} + +template +inline bool OpIter::readStore(ValType resultType, uint32_t byteSize, + LinearMemoryAddress* addr, + Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::Store); + + if (!popWithType(resultType, value)) { + return false; + } + + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readTeeStore(ValType resultType, uint32_t byteSize, + LinearMemoryAddress* addr, + Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::TeeStore); + + if (!popWithType(resultType, value)) { + return false; + } + + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + infalliblePush(TypeAndValue(resultType, *value)); + return true; +} + +template +inline bool OpIter::readNop() { + MOZ_ASSERT(Classify(op_) == OpKind::Nop); + + return true; +} + +template +inline bool OpIter::readMemorySize() { + MOZ_ASSERT(Classify(op_) == OpKind::MemorySize); + + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + + uint8_t flags; + if (!readFixedU8(&flags)) { + return fail("failed to read memory flags"); + } + + if (flags != uint8_t(0)) { + return fail("unexpected flags"); + } + + ValType ptrType = ToValType(env_.memory->indexType()); + return push(ptrType); +} + +template +inline bool OpIter::readMemoryGrow(Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::MemoryGrow); + + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + + uint8_t flags; + if (!readFixedU8(&flags)) { + return fail("failed to read memory flags"); + } + + if (flags != uint8_t(0)) { + return fail("unexpected flags"); + } + + ValType ptrType = ToValType(env_.memory->indexType()); + if (!popWithType(ptrType, input)) { + return false; + } + + infalliblePush(ptrType); + + return true; +} + +template +inline bool OpIter::readSelect(bool typed, StackType* type, + Value* trueValue, Value* falseValue, + Value* condition) { + MOZ_ASSERT(Classify(op_) == OpKind::Select); + + if (typed) { + uint32_t length; + if (!readVarU32(&length)) { + return fail("unable to read select result length"); + } + if (length != 1) { + return fail("bad number of results"); + } + ValType result; + if (!readValType(&result)) { + return fail("invalid result type for select"); + } + + if (!popWithType(ValType::I32, condition)) { + return false; + } + if (!popWithType(result, falseValue)) { + return false; + } + if (!popWithType(result, trueValue)) { + return false; + } + + *type = StackType(result); + infalliblePush(*type); + return true; + } + + if (!popWithType(ValType::I32, condition)) { + return false; + } + + StackType falseType; + if (!popStackType(&falseType, falseValue)) { + return false; + } + + StackType trueType; + if (!popStackType(&trueType, trueValue)) { + return false; + } + + if (!falseType.isValidForUntypedSelect() || + !trueType.isValidForUntypedSelect()) { + return fail("invalid types for untyped select"); + } + + if (falseType.isStackBottom()) { + *type = trueType; + } else if (trueType.isStackBottom() || falseType == trueType) { + *type = falseType; + } else { + return fail("select operand types must match"); + } + + infalliblePush(*type); + return true; +} + +template +inline bool OpIter::readGetLocal(const ValTypeVector& locals, + uint32_t* id) { + MOZ_ASSERT(Classify(op_) == OpKind::GetLocal); + + if (!readVarU32(id)) { + return fail("unable to read local index"); + } + + if (*id >= locals.length()) { + return fail("local.get index out of range"); + } + + if (unsetLocals_.isUnset(*id)) { + return fail("local.get read from unset local"); + } + + return push(locals[*id]); +} + +template +inline bool OpIter::readSetLocal(const ValTypeVector& locals, + uint32_t* id, Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::SetLocal); + + if (!readVarU32(id)) { + return fail("unable to read local index"); + } + + if (*id >= locals.length()) { + return fail("local.set index out of range"); + } + + if (unsetLocals_.isUnset(*id)) { + unsetLocals_.set(*id, controlStackDepth()); + } + + return popWithType(locals[*id], value); +} + +template +inline bool OpIter::readTeeLocal(const ValTypeVector& locals, + uint32_t* id, Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::TeeLocal); + + if (!readVarU32(id)) { + return fail("unable to read local index"); + } + + if (*id >= locals.length()) { + return fail("local.set index out of range"); + } + + if (unsetLocals_.isUnset(*id)) { + unsetLocals_.set(*id, controlStackDepth()); + } + + ValueVector single; + if (!checkTopTypeMatches(ResultType::Single(locals[*id]), &single, + /*rewriteStackTypes=*/true)) { + return false; + } + + *value = single[0]; + return true; +} + +template +inline bool OpIter::readGetGlobal(uint32_t* id) { + MOZ_ASSERT(Classify(op_) == OpKind::GetGlobal); + + if (!d_.readGlobalIndex(id)) { + return false; + } + + if (*id >= env_.globals.length()) { + return fail("global.get index out of range"); + } + + // Initializer expressions can access immutable imported globals, or any + // previously defined global with GC enabled. + if (kind_ == OpIter::InitExpr && *id >= maxInitializedGlobalsIndexPlus1_ && + (!env_.globals[*id].isImport() || env_.globals[*id].isMutable())) { + return fail( + "global.get in initializer expression must reference a global " + "immutable import"); + } + + return push(env_.globals[*id].type()); +} + +template +inline bool OpIter::readSetGlobal(uint32_t* id, Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::SetGlobal); + + if (!d_.readGlobalIndex(id)) { + return false; + } + + if (*id >= env_.globals.length()) { + return fail("global.set index out of range"); + } + + if (!env_.globals[*id].isMutable()) { + return fail("can't write an immutable global"); + } + + return popWithType(env_.globals[*id].type(), value); +} + +template +inline bool OpIter::readTeeGlobal(uint32_t* id, Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::TeeGlobal); + + if (!d_.readGlobalIndex(id)) { + return false; + } + + if (*id >= env_.globals.length()) { + return fail("global.set index out of range"); + } + + if (!env_.globals[*id].isMutable()) { + return fail("can't write an immutable global"); + } + + ValueVector single; + if (!checkTopTypeMatches(ResultType::Single(env_.globals[*id].type()), + &single, + /*rewriteStackTypes=*/true)) { + return false; + } + + MOZ_ASSERT(single.length() == 1); + *value = single[0]; + return true; +} + +template +inline bool OpIter::readI32Const(int32_t* i32) { + MOZ_ASSERT(Classify(op_) == OpKind::I32); + + if (!d_.readI32Const(i32)) { + return false; + } + + return push(ValType::I32); +} + +template +inline bool OpIter::readI64Const(int64_t* i64) { + MOZ_ASSERT(Classify(op_) == OpKind::I64); + + if (!d_.readI64Const(i64)) { + return false; + } + + return push(ValType::I64); +} + +template +inline bool OpIter::readF32Const(float* f32) { + MOZ_ASSERT(Classify(op_) == OpKind::F32); + + if (!d_.readF32Const(f32)) { + return false; + } + + return push(ValType::F32); +} + +template +inline bool OpIter::readF64Const(double* f64) { + MOZ_ASSERT(Classify(op_) == OpKind::F64); + + if (!d_.readF64Const(f64)) { + return false; + } + + return push(ValType::F64); +} + +template +inline bool OpIter::readRefFunc(uint32_t* funcIndex) { + MOZ_ASSERT(Classify(op_) == OpKind::RefFunc); + + if (!d_.readFuncIndex(funcIndex)) { + return false; + } + if (*funcIndex >= env_.funcs.length()) { + return fail("function index out of range"); + } + if (kind_ == OpIter::Func && !env_.funcs[*funcIndex].canRefFunc()) { + return fail( + "function index is not declared in a section before the code section"); + } + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + // When function references enabled, push type index on the stack, e.g. for + // validation of the call_ref instruction. + if (env_.functionReferencesEnabled()) { + const uint32_t typeIndex = env_.funcs[*funcIndex].typeIndex; + const TypeDef& typeDef = env_.types->type(typeIndex); + return push(RefType::fromTypeDef(&typeDef, false)); + } +#endif + return push(RefType::func()); +} + +template +inline bool OpIter::readRefNull(RefType* type) { + MOZ_ASSERT(Classify(op_) == OpKind::RefNull); + + if (!d_.readRefNull(*env_.types, env_.features, type)) { + return false; + } + return push(*type); +} + +template +inline bool OpIter::readRefIsNull(Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::Conversion); + + StackType type; + if (!popWithRefType(input, &type)) { + return false; + } + return push(ValType::I32); +} + +template +inline bool OpIter::readRefAsNonNull(Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::RefAsNonNull); + + StackType type; + if (!popWithRefType(input, &type)) { + return false; + } + + if (type.isStackBottom()) { + infalliblePush(type); + } else { + infalliblePush(TypeAndValue(type.asNonNullable(), *input)); + } + return true; +} + +template +inline bool OpIter::readBrOnNull(uint32_t* relativeDepth, + ResultType* type, ValueVector* values, + Value* condition) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnNull); + + if (!readVarU32(relativeDepth)) { + return fail("unable to read br_on_null depth"); + } + + StackType refType; + if (!popWithRefType(condition, &refType)) { + return false; + } + + if (!checkBranchValueAndPush(*relativeDepth, type, values)) { + return false; + } + + if (refType.isStackBottom()) { + return push(refType); + } + return push(TypeAndValue(refType.asNonNullable(), *condition)); +} + +template +inline bool OpIter::readBrOnNonNull(uint32_t* relativeDepth, + ResultType* type, + ValueVector* values, + Value* condition) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnNonNull); + + if (!readVarU32(relativeDepth)) { + return fail("unable to read br_on_non_null depth"); + } + + Control* block = nullptr; + if (!getControl(*relativeDepth, &block)) { + return false; + } + + *type = block->branchTargetType(); + + // Check we at least have one type in the branch target type. + if (type->length() < 1) { + return fail("type mismatch: target block type expected to be [_, ref]"); + } + + // Pop the condition reference. + StackType refType; + if (!popWithRefType(condition, &refType)) { + return false; + } + + // Push non-nullable version of condition reference on the stack, prior + // checking the target type below. + if (!(refType.isStackBottom() + ? push(refType) + : push(TypeAndValue(refType.asNonNullable(), *condition)))) { + return false; + } + + // Check if the type stack matches the branch target type. + if (!checkTopTypeMatches(*type, values, /*rewriteStackTypes=*/false)) { + return false; + } + + // Pop the condition reference -- the null-branch does not receive the value. + StackType unusedType; + Value unusedValue; + return popStackType(&unusedType, &unusedValue); +} + +template +inline bool OpIter::popCallArgs(const ValTypeVector& expectedTypes, + ValueVector* values) { + // Iterate through the argument types backward so that pops occur in the + // right order. + + if (!values->resize(expectedTypes.length())) { + return false; + } + + for (int32_t i = int32_t(expectedTypes.length()) - 1; i >= 0; i--) { + if (!popWithType(expectedTypes[i], &(*values)[i])) { + return false; + } + } + + return true; +} + +template +inline bool OpIter::readCall(uint32_t* funcTypeIndex, + ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::Call); + + if (!readVarU32(funcTypeIndex)) { + return fail("unable to read call function index"); + } + + if (*funcTypeIndex >= env_.funcs.length()) { + return fail("callee index out of range"); + } + + const FuncType& funcType = *env_.funcs[*funcTypeIndex].type; + + if (!popCallArgs(funcType.args(), argValues)) { + return false; + } + + return push(ResultType::Vector(funcType.results())); +} + +template +inline bool OpIter::readCallIndirect(uint32_t* funcTypeIndex, + uint32_t* tableIndex, + Value* callee, + ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::CallIndirect); + MOZ_ASSERT(funcTypeIndex != tableIndex); + + if (!readVarU32(funcTypeIndex)) { + return fail("unable to read call_indirect signature index"); + } + + if (*funcTypeIndex >= env_.numTypes()) { + return fail("signature index out of range"); + } + + if (!readVarU32(tableIndex)) { + return fail("unable to read call_indirect table index"); + } + if (*tableIndex >= env_.tables.length()) { + // Special case this for improved user experience. + if (!env_.tables.length()) { + return fail("can't call_indirect without a table"); + } + return fail("table index out of range for call_indirect"); + } + if (!env_.tables[*tableIndex].elemType.isFuncHierarchy()) { + return fail("indirect calls must go through a table of 'funcref'"); + } + + if (!popWithType(ValType::I32, callee)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*funcTypeIndex); + if (!typeDef.isFuncType()) { + return fail("expected signature type"); + } + const FuncType& funcType = typeDef.funcType(); + + if (!popCallArgs(funcType.args(), argValues)) { + return false; + } + + return push(ResultType::Vector(funcType.results())); +} + +#ifdef ENABLE_WASM_FUNCTION_REFERENCES +template +inline bool OpIter::readCallRef(const FuncType** funcType, + Value* callee, ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::CallRef); + + uint32_t funcTypeIndex; + if (!readFuncTypeIndex(&funcTypeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(funcTypeIndex); + *funcType = &typeDef.funcType(); + + if (!popWithType(ValType(RefType::fromTypeDef(&typeDef, true)), callee)) { + return false; + } + + if (!popCallArgs((*funcType)->args(), argValues)) { + return false; + } + + return push(ResultType::Vector((*funcType)->results())); +} +#endif + +template +inline bool OpIter::readOldCallDirect(uint32_t numFuncImports, + uint32_t* funcTypeIndex, + ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::OldCallDirect); + + uint32_t funcDefIndex; + if (!readVarU32(&funcDefIndex)) { + return fail("unable to read call function index"); + } + + if (UINT32_MAX - funcDefIndex < numFuncImports) { + return fail("callee index out of range"); + } + + *funcTypeIndex = numFuncImports + funcDefIndex; + + if (*funcTypeIndex >= env_.funcs.length()) { + return fail("callee index out of range"); + } + + const FuncType& funcType = *env_.funcs[*funcTypeIndex].type; + + if (!popCallArgs(funcType.args(), argValues)) { + return false; + } + + return push(ResultType::Vector(funcType.results())); +} + +template +inline bool OpIter::readOldCallIndirect(uint32_t* funcTypeIndex, + Value* callee, + ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::OldCallIndirect); + + if (!readVarU32(funcTypeIndex)) { + return fail("unable to read call_indirect signature index"); + } + + if (*funcTypeIndex >= env_.numTypes()) { + return fail("signature index out of range"); + } + + const TypeDef& typeDef = env_.types->type(*funcTypeIndex); + if (!typeDef.isFuncType()) { + return fail("expected signature type"); + } + const FuncType& funcType = typeDef.funcType(); + + if (!popCallArgs(funcType.args(), argValues)) { + return false; + } + + if (!popWithType(ValType::I32, callee)) { + return false; + } + + return push(ResultType::Vector(funcType.results())); +} + +template +inline bool OpIter::readWake(LinearMemoryAddress* addr, + Value* count) { + MOZ_ASSERT(Classify(op_) == OpKind::Wake); + + if (!popWithType(ValType::I32, count)) { + return false; + } + + uint32_t byteSize = 4; // Per spec; smallest WAIT is i32. + + if (!readLinearMemoryAddressAligned(byteSize, addr)) { + return false; + } + + infalliblePush(ValType::I32); + return true; +} + +template +inline bool OpIter::readWait(LinearMemoryAddress* addr, + ValType valueType, uint32_t byteSize, + Value* value, Value* timeout) { + MOZ_ASSERT(Classify(op_) == OpKind::Wait); + + if (!popWithType(ValType::I64, timeout)) { + return false; + } + + if (!popWithType(valueType, value)) { + return false; + } + + if (!readLinearMemoryAddressAligned(byteSize, addr)) { + return false; + } + + infalliblePush(ValType::I32); + return true; +} + +template +inline bool OpIter::readFence() { + MOZ_ASSERT(Classify(op_) == OpKind::Fence); + uint8_t flags; + if (!readFixedU8(&flags)) { + return fail("expected memory order after fence"); + } + if (flags != 0) { + return fail("non-zero memory order not supported yet"); + } + return true; +} + +template +inline bool OpIter::readAtomicLoad(LinearMemoryAddress* addr, + ValType resultType, + uint32_t byteSize) { + MOZ_ASSERT(Classify(op_) == OpKind::AtomicLoad); + + if (!readLinearMemoryAddressAligned(byteSize, addr)) { + return false; + } + + infalliblePush(resultType); + return true; +} + +template +inline bool OpIter::readAtomicStore(LinearMemoryAddress* addr, + ValType resultType, + uint32_t byteSize, Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::AtomicStore); + + if (!popWithType(resultType, value)) { + return false; + } + + if (!readLinearMemoryAddressAligned(byteSize, addr)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readAtomicRMW(LinearMemoryAddress* addr, + ValType resultType, uint32_t byteSize, + Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::AtomicBinOp); + + if (!popWithType(resultType, value)) { + return false; + } + + if (!readLinearMemoryAddressAligned(byteSize, addr)) { + return false; + } + + infalliblePush(resultType); + return true; +} + +template +inline bool OpIter::readAtomicCmpXchg(LinearMemoryAddress* addr, + ValType resultType, + uint32_t byteSize, + Value* oldValue, + Value* newValue) { + MOZ_ASSERT(Classify(op_) == OpKind::AtomicCompareExchange); + + if (!popWithType(resultType, newValue)) { + return false; + } + + if (!popWithType(resultType, oldValue)) { + return false; + } + + if (!readLinearMemoryAddressAligned(byteSize, addr)) { + return false; + } + + infalliblePush(resultType); + return true; +} + +template +inline bool OpIter::readMemOrTableCopy(bool isMem, + uint32_t* dstMemOrTableIndex, + Value* dst, + uint32_t* srcMemOrTableIndex, + Value* src, Value* len) { + MOZ_ASSERT(Classify(op_) == OpKind::MemOrTableCopy); + MOZ_ASSERT(dstMemOrTableIndex != srcMemOrTableIndex); + + // Spec requires (dest, src) as of 2019-10-04. + if (!readMemOrTableIndex(isMem, dstMemOrTableIndex)) { + return false; + } + if (!readMemOrTableIndex(isMem, srcMemOrTableIndex)) { + return false; + } + + if (isMem) { + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + if (*srcMemOrTableIndex != 0 || *dstMemOrTableIndex != 0) { + return fail("memory index out of range for memory.copy"); + } + } else { + if (*dstMemOrTableIndex >= env_.tables.length() || + *srcMemOrTableIndex >= env_.tables.length()) { + return fail("table index out of range for table.copy"); + } + ValType dstElemType = env_.tables[*dstMemOrTableIndex].elemType; + ValType srcElemType = env_.tables[*srcMemOrTableIndex].elemType; + if (!checkIsSubtypeOf(srcElemType, dstElemType)) { + return false; + } + } + + ValType ptrType = isMem ? ToValType(env_.memory->indexType()) : ValType::I32; + + if (!popWithType(ptrType, len)) { + return false; + } + + if (!popWithType(ptrType, src)) { + return false; + } + + if (!popWithType(ptrType, dst)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readDataOrElemDrop(bool isData, + uint32_t* segIndex) { + MOZ_ASSERT(Classify(op_) == OpKind::DataOrElemDrop); + + if (!readVarU32(segIndex)) { + return fail("unable to read segment index"); + } + + if (isData) { + if (env_.dataCount.isNothing()) { + return fail("data.drop requires a DataCount section"); + } + if (*segIndex >= *env_.dataCount) { + return fail("data.drop segment index out of range"); + } + } else { + if (*segIndex >= env_.elemSegments.length()) { + return fail("element segment index out of range for elem.drop"); + } + } + + return true; +} + +template +inline bool OpIter::readMemFill(Value* start, Value* val, Value* len) { + MOZ_ASSERT(Classify(op_) == OpKind::MemFill); + + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + + uint8_t memoryIndex; + if (!readFixedU8(&memoryIndex)) { + return fail("failed to read memory index"); + } + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + if (memoryIndex != 0) { + return fail("memory index must be zero"); + } + + ValType ptrType = ToValType(env_.memory->indexType()); + + if (!popWithType(ptrType, len)) { + return false; + } + + if (!popWithType(ValType::I32, val)) { + return false; + } + + if (!popWithType(ptrType, start)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readMemOrTableInit(bool isMem, uint32_t* segIndex, + uint32_t* dstTableIndex, + Value* dst, Value* src, + Value* len) { + MOZ_ASSERT(Classify(op_) == OpKind::MemOrTableInit); + MOZ_ASSERT(segIndex != dstTableIndex); + + if (!readVarU32(segIndex)) { + return fail("unable to read segment index"); + } + + uint32_t memOrTableIndex = 0; + if (!readMemOrTableIndex(isMem, &memOrTableIndex)) { + return false; + } + + if (isMem) { + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + if (memOrTableIndex != 0) { + return fail("memory index must be zero"); + } + if (env_.dataCount.isNothing()) { + return fail("memory.init requires a DataCount section"); + } + if (*segIndex >= *env_.dataCount) { + return fail("memory.init segment index out of range"); + } + } else { + if (memOrTableIndex >= env_.tables.length()) { + return fail("table index out of range for table.init"); + } + *dstTableIndex = memOrTableIndex; + + if (*segIndex >= env_.elemSegments.length()) { + return fail("table.init segment index out of range"); + } + if (!checkIsSubtypeOf(env_.elemSegments[*segIndex]->elemType, + env_.tables[*dstTableIndex].elemType)) { + return false; + } + } + + if (!popWithType(ValType::I32, len)) { + return false; + } + + if (!popWithType(ValType::I32, src)) { + return false; + } + + ValType ptrType = isMem ? ToValType(env_.memory->indexType()) : ValType::I32; + return popWithType(ptrType, dst); +} + +template +inline bool OpIter::readTableFill(uint32_t* tableIndex, Value* start, + Value* val, Value* len) { + MOZ_ASSERT(Classify(op_) == OpKind::TableFill); + + if (!readVarU32(tableIndex)) { + return fail("unable to read table index"); + } + if (*tableIndex >= env_.tables.length()) { + return fail("table index out of range for table.fill"); + } + + if (!popWithType(ValType::I32, len)) { + return false; + } + if (!popWithType(env_.tables[*tableIndex].elemType, val)) { + return false; + } + if (!popWithType(ValType::I32, start)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readMemDiscard(Value* start, Value* len) { + MOZ_ASSERT(Classify(op_) == OpKind::MemDiscard); + + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + + uint8_t memoryIndex; + if (!readFixedU8(&memoryIndex)) { + return fail("failed to read memory index"); + } + if (memoryIndex != 0) { + return fail("memory index must be zero"); + } + + ValType ptrType = ToValType(env_.memory->indexType()); + + if (!popWithType(ptrType, len)) { + return false; + } + + if (!popWithType(ptrType, start)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readTableGet(uint32_t* tableIndex, Value* index) { + MOZ_ASSERT(Classify(op_) == OpKind::TableGet); + + if (!readVarU32(tableIndex)) { + return fail("unable to read table index"); + } + if (*tableIndex >= env_.tables.length()) { + return fail("table index out of range for table.get"); + } + + if (!popWithType(ValType::I32, index)) { + return false; + } + + infalliblePush(env_.tables[*tableIndex].elemType); + return true; +} + +template +inline bool OpIter::readTableGrow(uint32_t* tableIndex, + Value* initValue, Value* delta) { + MOZ_ASSERT(Classify(op_) == OpKind::TableGrow); + + if (!readVarU32(tableIndex)) { + return fail("unable to read table index"); + } + if (*tableIndex >= env_.tables.length()) { + return fail("table index out of range for table.grow"); + } + + if (!popWithType(ValType::I32, delta)) { + return false; + } + if (!popWithType(env_.tables[*tableIndex].elemType, initValue)) { + return false; + } + + infalliblePush(ValType::I32); + return true; +} + +template +inline bool OpIter::readTableSet(uint32_t* tableIndex, Value* index, + Value* value) { + MOZ_ASSERT(Classify(op_) == OpKind::TableSet); + + if (!readVarU32(tableIndex)) { + return fail("unable to read table index"); + } + if (*tableIndex >= env_.tables.length()) { + return fail("table index out of range for table.set"); + } + + if (!popWithType(env_.tables[*tableIndex].elemType, value)) { + return false; + } + if (!popWithType(ValType::I32, index)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readTableSize(uint32_t* tableIndex) { + MOZ_ASSERT(Classify(op_) == OpKind::TableSize); + + *tableIndex = 0; + + if (!readVarU32(tableIndex)) { + return fail("unable to read table index"); + } + if (*tableIndex >= env_.tables.length()) { + return fail("table index out of range for table.size"); + } + + return push(ValType::I32); +} + +template +inline bool OpIter::readGcTypeIndex(uint32_t* typeIndex) { + if (!d_.readTypeIndex(typeIndex)) { + return false; + } + + if (*typeIndex >= env_.types->length()) { + return fail("type index out of range"); + } + + if (!env_.types->type(*typeIndex).isStructType() && + !env_.types->type(*typeIndex).isArrayType()) { + return fail("not a gc type"); + } + + return true; +} + +template +inline bool OpIter::readStructTypeIndex(uint32_t* typeIndex) { + if (!readVarU32(typeIndex)) { + return fail("unable to read type index"); + } + + if (*typeIndex >= env_.types->length()) { + return fail("type index out of range"); + } + + if (!env_.types->type(*typeIndex).isStructType()) { + return fail("not a struct type"); + } + + return true; +} + +template +inline bool OpIter::readArrayTypeIndex(uint32_t* typeIndex) { + if (!readVarU32(typeIndex)) { + return fail("unable to read type index"); + } + + if (*typeIndex >= env_.types->length()) { + return fail("type index out of range"); + } + + if (!env_.types->type(*typeIndex).isArrayType()) { + return fail("not an array type"); + } + + return true; +} + +template +inline bool OpIter::readFuncTypeIndex(uint32_t* typeIndex) { + if (!readVarU32(typeIndex)) { + return fail("unable to read type index"); + } + + if (*typeIndex >= env_.types->length()) { + return fail("type index out of range"); + } + + if (!env_.types->type(*typeIndex).isFuncType()) { + return fail("not an func type"); + } + + return true; +} + +template +inline bool OpIter::readFieldIndex(uint32_t* fieldIndex, + const StructType& structType) { + if (!readVarU32(fieldIndex)) { + return fail("unable to read field index"); + } + + if (structType.fields_.length() <= *fieldIndex) { + return fail("field index out of range"); + } + + return true; +} + +#ifdef ENABLE_WASM_GC + +template +inline bool OpIter::readStructNew(uint32_t* typeIndex, + ValueVector* argValues) { + MOZ_ASSERT(Classify(op_) == OpKind::StructNew); + + if (!readStructTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const StructType& structType = typeDef.structType(); + + if (!argValues->resize(structType.fields_.length())) { + return false; + } + + static_assert(MaxStructFields <= INT32_MAX, "Or we iloop below"); + + for (int32_t i = structType.fields_.length() - 1; i >= 0; i--) { + if (!popWithType(structType.fields_[i].type.widenToValType(), + &(*argValues)[i])) { + return false; + } + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readStructNewDefault(uint32_t* typeIndex) { + MOZ_ASSERT(Classify(op_) == OpKind::StructNewDefault); + + if (!readStructTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const StructType& structType = typeDef.structType(); + + if (!structType.isDefaultable()) { + return fail("struct must be defaultable"); + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readStructGet(uint32_t* typeIndex, + uint32_t* fieldIndex, + FieldWideningOp wideningOp, + Value* ptr) { + MOZ_ASSERT(typeIndex != fieldIndex); + MOZ_ASSERT(Classify(op_) == OpKind::StructGet); + + if (!readStructTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const StructType& structType = typeDef.structType(); + + if (!readFieldIndex(fieldIndex, structType)) { + return false; + } + + if (!popWithType(RefType::fromTypeDef(&typeDef, true), ptr)) { + return false; + } + + FieldType fieldType = structType.fields_[*fieldIndex].type; + + if (fieldType.isValType() && wideningOp != FieldWideningOp::None) { + return fail("must not specify signedness for unpacked field type"); + } + + if (!fieldType.isValType() && wideningOp == FieldWideningOp::None) { + return fail("must specify signedness for packed field type"); + } + + return push(fieldType.widenToValType()); +} + +template +inline bool OpIter::readStructSet(uint32_t* typeIndex, + uint32_t* fieldIndex, Value* ptr, + Value* val) { + MOZ_ASSERT(typeIndex != fieldIndex); + MOZ_ASSERT(Classify(op_) == OpKind::StructSet); + + if (!readStructTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const StructType& structType = typeDef.structType(); + + if (!readFieldIndex(fieldIndex, structType)) { + return false; + } + + if (!popWithType(structType.fields_[*fieldIndex].type.widenToValType(), + val)) { + return false; + } + + if (!structType.fields_[*fieldIndex].isMutable) { + return fail("field is not mutable"); + } + + if (!popWithType(RefType::fromTypeDef(&typeDef, true), ptr)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readArrayNew(uint32_t* typeIndex, + Value* numElements, Value* argValue) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayNew); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + + if (!popWithType(ValType::I32, numElements)) { + return false; + } + + if (!popWithType(arrayType.elementType_.widenToValType(), argValue)) { + return false; + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readArrayNewFixed(uint32_t* typeIndex, + uint32_t* numElements, + ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayNewFixed); + MOZ_ASSERT(values->length() == 0); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + + if (!readVarU32(numElements)) { + return false; + } + // Don't resize `values` so as to hold `numElements`. If `numElements` is + // absurdly large, this will will take a large amount of time and memory, + // which will be wasted because `popWithType` in the loop below will soon + // start failing anyway. + + ValType widenedElementType = arrayType.elementType_.widenToValType(); + for (uint32_t i = 0; i < *numElements; i++) { + Value v; + if (!popWithType(widenedElementType, &v)) { + return false; + } + if (!values->append(v)) { + return false; + } + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readArrayNewDefault(uint32_t* typeIndex, + Value* numElements) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayNewDefault); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + + if (!popWithType(ValType::I32, numElements)) { + return false; + } + + if (!arrayType.elementType_.isDefaultable()) { + return fail("array must be defaultable"); + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readArrayNewData(uint32_t* typeIndex, + uint32_t* segIndex, Value* offset, + Value* numElements) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayNewData); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + if (!readVarU32(segIndex)) { + return fail("unable to read segment index"); + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + FieldType elemType = arrayType.elementType_; + if (!elemType.isNumber() && !elemType.isPacked() && !elemType.isVector()) { + return fail("element type must be i8/i16/i32/i64/f32/f64/v128"); + } + if (env_.dataCount.isNothing()) { + return fail("datacount section missing"); + } + if (*segIndex >= *env_.dataCount) { + return fail("segment index is out of range"); + } + + if (!popWithType(ValType::I32, numElements)) { + return false; + } + if (!popWithType(ValType::I32, offset)) { + return false; + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readArrayNewElem(uint32_t* typeIndex, + uint32_t* segIndex, Value* offset, + Value* numElements) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayNewData); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + if (!readVarU32(segIndex)) { + return fail("unable to read segment index"); + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + FieldType dstElemType = arrayType.elementType_; + if (!dstElemType.isRefType()) { + return fail("element type is not a reftype"); + } + if (*segIndex >= env_.elemSegments.length()) { + return fail("segment index is out of range"); + } + + const ElemSegment* elemSeg = env_.elemSegments[*segIndex]; + RefType srcElemType = elemSeg->elemType; + // srcElemType needs to be a subtype (child) of dstElemType + if (!checkIsSubtypeOf(srcElemType, dstElemType.refType())) { + return fail("incompatible element types"); + } + + if (!popWithType(ValType::I32, numElements)) { + return false; + } + if (!popWithType(ValType::I32, offset)) { + return false; + } + + return push(RefType::fromTypeDef(&typeDef, false)); +} + +template +inline bool OpIter::readArrayGet(uint32_t* typeIndex, + FieldWideningOp wideningOp, + Value* index, Value* ptr) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayGet); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + + if (!popWithType(ValType::I32, index)) { + return false; + } + + if (!popWithType(RefType::fromTypeDef(&typeDef, true), ptr)) { + return false; + } + + FieldType fieldType = arrayType.elementType_; + + if (fieldType.isValType() && wideningOp != FieldWideningOp::None) { + return fail("must not specify signedness for unpacked element type"); + } + + if (!fieldType.isValType() && wideningOp == FieldWideningOp::None) { + return fail("must specify signedness for packed element type"); + } + + return push(fieldType.widenToValType()); +} + +template +inline bool OpIter::readArraySet(uint32_t* typeIndex, Value* val, + Value* index, Value* ptr) { + MOZ_ASSERT(Classify(op_) == OpKind::ArraySet); + + if (!readArrayTypeIndex(typeIndex)) { + return false; + } + + const TypeDef& typeDef = env_.types->type(*typeIndex); + const ArrayType& arrayType = typeDef.arrayType(); + + if (!arrayType.isMutable_) { + return fail("array is not mutable"); + } + + if (!popWithType(arrayType.elementType_.widenToValType(), val)) { + return false; + } + + if (!popWithType(ValType::I32, index)) { + return false; + } + + if (!popWithType(RefType::fromTypeDef(&typeDef, true), ptr)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readArrayLen(bool decodeIgnoredTypeIndex, + Value* ptr) { + MOZ_ASSERT(Classify(op_) == OpKind::ArrayLen); + + // TODO: remove once V8 removes array.len with type index from their snapshot + uint32_t unused; + if (decodeIgnoredTypeIndex && !readVarU32(&unused)) { + return false; + } + + if (!popWithType(RefType::array(), ptr)) { + return false; + } + + return push(ValType::I32); +} + +template +inline bool OpIter::readArrayCopy(int32_t* elemSize, + bool* elemsAreRefTyped, + Value* dstArray, Value* dstIndex, + Value* srcArray, Value* srcIndex, + Value* numElements) { + // *elemSize is set to 1/2/4/8/16, and *elemsAreRefTyped is set to indicate + // *ref-typeness of elements. + MOZ_ASSERT(Classify(op_) == OpKind::ArrayCopy); + + uint32_t dstTypeIndex, srcTypeIndex; + if (!readArrayTypeIndex(&dstTypeIndex)) { + return false; + } + if (!readArrayTypeIndex(&srcTypeIndex)) { + return false; + } + + // `dstTypeIndex`/`srcTypeIndex` are ensured by the above to both be array + // types. Reject if: + // * the dst array is not of mutable type + // * the element types are incompatible + const TypeDef& dstTypeDef = env_.types->type(dstTypeIndex); + const ArrayType& dstArrayType = dstTypeDef.arrayType(); + const TypeDef& srcTypeDef = env_.types->type(srcTypeIndex); + const ArrayType& srcArrayType = srcTypeDef.arrayType(); + FieldType dstElemType = dstArrayType.elementType_; + FieldType srcElemType = srcArrayType.elementType_; + if (!dstArrayType.isMutable_) { + return fail("destination array is not mutable"); + } + + if (!checkIsSubtypeOf(srcElemType, dstElemType)) { + return fail("incompatible element types"); + } + bool dstIsRefType = dstElemType.isRefType(); + MOZ_ASSERT(dstIsRefType == srcElemType.isRefType()); + + *elemSize = int32_t(dstElemType.size()); + *elemsAreRefTyped = dstIsRefType; + MOZ_ASSERT(*elemSize >= 1 && *elemSize <= 16); + MOZ_ASSERT_IF(*elemsAreRefTyped, *elemSize == 4 || *elemSize == 8); + + if (!popWithType(ValType::I32, numElements)) { + return false; + } + if (!popWithType(ValType::I32, srcIndex)) { + return false; + } + if (!popWithType(RefType::fromTypeDef(&srcTypeDef, true), srcArray)) { + return false; + } + if (!popWithType(ValType::I32, dstIndex)) { + return false; + } + if (!popWithType(RefType::fromTypeDef(&dstTypeDef, true), dstArray)) { + return false; + } + + return true; +} + +template +inline bool OpIter::readRefTestV5(RefType* sourceType, + uint32_t* typeIndex, Value* ref) { + MOZ_ASSERT(Classify(op_) == OpKind::RefTestV5); + + if (!readGcTypeIndex(typeIndex)) { + return false; + } + + StackType inputType; + if (!popWithType(RefType::any(), ref)) { + return false; + } + *sourceType = inputType.valTypeOr(RefType::any()).refType(); + + return push(ValType(ValType::I32)); +} + +template +inline bool OpIter::readRefCastV5(RefType* sourceType, + uint32_t* typeIndex, Value* ref) { + MOZ_ASSERT(Classify(op_) == OpKind::RefCastV5); + + if (!readGcTypeIndex(typeIndex)) { + return false; + } + + StackType inputType; + if (!popWithType(RefType::any(), ref, &inputType)) { + return false; + } + *sourceType = inputType.valTypeOr(RefType::any()).refType(); + + const TypeDef& typeDef = env_.types->type(*typeIndex); + return push(RefType::fromTypeDef(&typeDef, inputType.isNullableAsOperand())); +} + +template +inline bool OpIter::readRefTest(bool nullable, RefType* sourceType, + RefType* destType, Value* ref) { + MOZ_ASSERT(Classify(op_) == OpKind::RefTest); + + if (!readHeapType(nullable, destType)) { + return false; + } + + StackType inputType; + if (!popWithType(destType->topType(), ref, &inputType)) { + return false; + } + *sourceType = inputType.valTypeOr(RefType::any()).refType(); + + if (!destType->isAnyHierarchy()) { + return fail("ref.test only supports the any hierarchy for now"); + } + + return push(ValType(ValType::I32)); +} + +template +inline bool OpIter::readRefCast(bool nullable, RefType* sourceType, + RefType* destType, Value* ref) { + MOZ_ASSERT(Classify(op_) == OpKind::RefCast); + + if (!readHeapType(nullable, destType)) { + return false; + } + + StackType inputType; + if (!popWithType(destType->topType(), ref, &inputType)) { + return false; + } + *sourceType = inputType.valTypeOr(RefType::any()).refType(); + + if (!destType->isAnyHierarchy()) { + return fail("ref.cast only supports the any hierarchy for now"); + } + + return push(*destType); +} + +// `br_on_cast ` +// branches if a reference has a given heap type. +// +// V6 spec text follows - note that br_on_cast and br_on_cast_fail are both +// handled by this function (disambiguated by a flag). +// +// * `br_on_cast ` branches if a reference has a +// given type +// - `br_on_cast $l rt1 rt2 : [t0* rt1] -> [t0* rt1\rt2]` +// - iff `$l : [t0* rt2]` +// - and `rt2 <: rt1` +// - passes operand along with branch under target type, plus possible extra +// args +// - if `rt2` contains `null`, branches on null, otherwise does not +// * `br_on_cast_fail ` branches if a reference +// does not have a given type +// - `br_on_cast_fail $l rt1 rt2 : [t0* rt1] -> [t0* rt2]` +// - iff `$l : [t0* rt1\rt2]` +// - and `rt2 <: rt1` +// - passes operand along with branch, plus possible extra args +// - if `rt2` contains `null`, does not branch on null, otherwise does +// where: +// - `(ref null1? ht1)\(ref null ht2) = (ref ht1)` +// - `(ref null1? ht1)\(ref ht2) = (ref null1? ht1)` +// +// The `rt1\rt2` syntax is a "diff" - it is basically rt1 minus rt2, because a +// successful cast to rt2 will branch away. So if rt2 allows null, the result +// after a non-branch will be non-null; on the other hand, if rt2 is +// non-nullable, the cast will have nothing to say about nullability and the +// nullability of rt1 will be preserved. +// +// `values` will be nonempty after the call, and its last entry will be the +// type that causes a branch (rt1\rt2 or rt2, depending). + +template +inline bool OpIter::readBrOnCast(bool* onSuccess, + uint32_t* labelRelativeDepth, + RefType* sourceType, RefType* destType, + ResultType* labelType, + ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnCast); + + uint8_t flags; + if (!readFixedU8(&flags)) { + return fail("unable to read br_on_cast flags"); + } + bool sourceNullable = flags & (1 << 0); + bool destNullable = flags & (1 << 1); + *onSuccess = !(flags & (1 << 2)); + + if (!readVarU32(labelRelativeDepth)) { + return fail("unable to read br_on_cast depth"); + } + + // This is distinct from the actual source type we pop from the stack, which + // can be more specific and allow for better optimizations. + RefType immediateSourceType; + if (!readHeapType(sourceNullable, &immediateSourceType)) { + return fail("unable to read br_on_cast source type"); + } + + if (!readHeapType(destNullable, destType)) { + return fail("unable to read br_on_cast dest type"); + } + + // Check that source and destination types are compatible + if (!checkIsSubtypeOf(*destType, immediateSourceType)) { + return fail( + "type mismatch: source and destination types for cast are " + "incompatible"); + } + + RefType typeOnSuccess = *destType; + // This is rt1\rt2 + RefType typeOnFail = + destNullable ? immediateSourceType.asNonNullable() : immediateSourceType; + RefType typeOnBranch = *onSuccess ? typeOnSuccess : typeOnFail; + RefType typeOnFallthrough = *onSuccess ? typeOnFail : typeOnSuccess; + + if (!typeOnSuccess.isAnyHierarchy() || !typeOnFail.isAnyHierarchy()) { + return fail( + "br_on_cast and br_on_cast_fail only support the any hierarchy for " + "now"); + } + + // Get the branch target type, which will also determine the type of extra + // values that are passed along on branch. + Control* block = nullptr; + if (!getControl(*labelRelativeDepth, &block)) { + return false; + } + *labelType = block->branchTargetType(); + + // Check we have at least one value slot in the branch target type, so as to + // receive the casted or non-casted type when we branch. + const size_t labelTypeNumValues = labelType->length(); + if (labelTypeNumValues < 1) { + return fail("type mismatch: branch target type has no value types"); + } + + // The last value slot in the branch target type is what is being cast. + // This slot is guaranteed to exist by the above check. + + // Check that the branch target type can accept typeOnBranch. + if (!checkIsSubtypeOf(typeOnBranch, (*labelType)[labelTypeNumValues - 1])) { + return false; + } + + // Replace the top operand with the result of falling through. Even branching + // on success can change the type on top of the stack on fallthrough. + Value inputValue; + StackType inputType; + if (!popWithType(immediateSourceType, &inputValue, &inputType)) { + return false; + } + *sourceType = inputType.valTypeOr(RefType::any()).refType(); + infalliblePush(TypeAndValue(typeOnFallthrough, inputValue)); + + // Create a copy of the branch target type, with the relevant value slot + // replaced by typeOnFallthrough. + ValTypeVector fallthroughTypes; + if (!labelType->cloneToVector(&fallthroughTypes)) { + return false; + } + fallthroughTypes[labelTypeNumValues - 1] = typeOnFallthrough; + + return checkTopTypeMatches(ResultType::Vector(fallthroughTypes), values, + /*rewriteStackTypes=*/false); +} + +template +inline bool OpIter::checkBrOnCastCommonV5(uint32_t labelRelativeDepth, + RefType* sourceType, + ValType castToType, + ResultType* labelType, + ValueVector* values) { + if (!(castToType.isRefType() && castToType.refType().isAnyHierarchy())) { + return fail("br_on_cast v5 only supports the any hierarchy"); + } + + // The casted from type is any subtype of anyref. + ValType anyrefType(RefType::any()); + *sourceType = RefType::any(); + + // Get the branch target type, which will also determine the type of extra + // values that are passed along with the casted type. This validates + // requirement (1). + Control* block = nullptr; + if (!getControl(labelRelativeDepth, &block)) { + return false; + } + *labelType = block->branchTargetType(); + + // Check we have at least one value slot in the branch target type, so as to + // receive the casted type in the case where the cast succeeds. + const size_t labelTypeNumValues = labelType->length(); + if (labelTypeNumValues < 1) { + return fail("type mismatch: branch target type has no value slots"); + } + + // The last value slot in the branch target type is what is being cast. + // This slot is guaranteed to exist by the above check. + + // Check that the branch target type can accept castType. The branch target + // may specify a supertype of castType, and this is okay. Validates (2). + if (!checkIsSubtypeOf(castToType, (*labelType)[labelTypeNumValues - 1])) { + return false; + } + + // Create a copy of the branch target type, with the relevant value slot + // replaced by anyrefType. Use this to check that the stack has the proper + // types to branch to the target type. + // + // TODO: We could avoid a potential allocation here by handwriting a custom + // checkTopTypeMatches that handles this case. + ValTypeVector fallthroughType; + if (!labelType->cloneToVector(&fallthroughType)) { + return false; + } + fallthroughType[labelTypeNumValues - 1] = anyrefType; + + // Validates the first half of (3), if we pretend that topType is eqref, + // which it isn't really. + return checkTopTypeMatches(ResultType::Vector(fallthroughType), values, + /*rewriteStackTypes=*/false); +} + +// `br_on_cast null? ` +// branches if a reference has a given heap type +// +// br_on_cast $label null? castType : [t0* (ref null argType)] -> +// [t0* (ref null2? argType)] +// (1) iff $label : [t0* labelType] +// (2) and (ref null3? castType) <: labelType +// (3) and castType <: topType and argType <: topType +// where topType is a common super type +// (4) and null? = null3? =/= null2? +// +// - passes operand along with branch under target type, +// plus possible extra args +// - if null? is present, branches on null, otherwise does not +// +// Currently unhandled: +// (3) partial check, and not really right +// (4) neither checked nor implemented +// +// `values` will be nonempty after the call, and its last entry will be that +// of the argument. + +template +inline bool OpIter::readBrOnCastV5(uint32_t* labelRelativeDepth, + RefType* sourceType, + uint32_t* castTypeIndex, + ResultType* labelType, + ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnCastV5); + + if (!readVarU32(labelRelativeDepth)) { + return fail("unable to read br_on_cast depth"); + } + + if (!readGcTypeIndex(castTypeIndex)) { + return false; + } + + // The casted to type is a non-nullable reference to the type index + // specified as an immediate. + const TypeDef& castTypeDef = env_.types->type(*castTypeIndex); + ValType castType(RefType::fromTypeDef(&castTypeDef, false)); + + return checkBrOnCastCommonV5(*labelRelativeDepth, sourceType, castType, + labelType, values); +} + +template +inline bool OpIter::readBrOnCastHeapV5( + bool nullable, uint32_t* labelRelativeDepth, RefType* sourceType, + RefType* destType, ResultType* labelType, ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnCastV5); + + if (!readVarU32(labelRelativeDepth)) { + return fail("unable to read br_on_cast depth"); + } + + if (!readHeapType(nullable, destType)) { + return false; + } + + ValType castToType(*destType); + return checkBrOnCastCommonV5(*labelRelativeDepth, sourceType, castToType, + labelType, values); +} + +// `br_on_cast_fail null? ` +// branches if a reference does not have a given heap type +// +// br_on_cast_fail $label null? castType : [t0* (ref null argType)] -> +// [t0* (ref null2? castType)] +// (1) iff $label : [t0* labelType] +// (2) and (ref null3? argType) <: labelType +// (3) and castType <: topType and argType <: topType +// where topType is a common super type +// (4) and null? = null2? =/= null3? +// +// - passes operand along with branch, plus possible extra args +// - if null? is present, does not branch on null, otherwise does +// +// Currently unhandled: +// (3) partial check, and not really right +// (4) neither checked nor implemented +// +// `values` will be nonempty after the call, and its last entry will be that +// of the argument. +template +inline bool OpIter::checkBrOnCastFailCommonV5( + uint32_t labelRelativeDepth, RefType* sourceType, ValType castToType, + ResultType* labelType, ValueVector* values) { + if (!(castToType.isRefType() && castToType.refType().isAnyHierarchy())) { + return fail("br_on_cast_fail v5 only supports the any hierarchy"); + } + + // Get the branch target type, which will also determine the type of extra + // values that are passed along with the casted type. This validates + // requirement (1). + Control* block = nullptr; + if (!getControl(labelRelativeDepth, &block)) { + return false; + } + *labelType = block->branchTargetType(); + + // Check we at least have one value slot in the branch target type, so as to + // receive the argument value in the case where the cast fails. + if (labelType->length() < 1) { + return fail("type mismatch: branch target type has no value slots"); + } + + // Check all operands match the failure label's target type. Validates (2). + if (!checkTopTypeMatches(*labelType, values, + /*rewriteStackTypes=*/false)) { + return false; + } + + // The top operand needs to be compatible with the casted from type. + // Validates the first half of (3), if we pretend that topType is eqref, + // which it isn't really. + Value ignored; + StackType inputValue; + if (!popWithType(ValType(RefType::any()), &ignored, &inputValue)) { + return false; + } + *sourceType = inputValue.valTypeOr(RefType::any()).refType(); + + // The top result in the fallthrough case is the casted to type. + infalliblePush(TypeAndValue(castToType, ignored)); + return true; +} + +template +inline bool OpIter::readBrOnCastFailV5(uint32_t* labelRelativeDepth, + RefType* sourceType, + uint32_t* castTypeIndex, + ResultType* labelType, + ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnCastFailV5); + + if (!readVarU32(labelRelativeDepth)) { + return fail("unable to read br_on_cast_fail depth"); + } + + if (!readGcTypeIndex(castTypeIndex)) { + return false; + } + + // The casted to type is a non-nullable reference to the type index + // specified as an immediate. + const TypeDef& castToTypeDef = env_.types->type(*castTypeIndex); + ValType castToType(RefType::fromTypeDef(&castToTypeDef, false)); + + return checkBrOnCastFailCommonV5(*labelRelativeDepth, sourceType, castToType, + labelType, values); +} + +template +inline bool OpIter::readBrOnCastFailHeapV5( + bool nullable, uint32_t* labelRelativeDepth, RefType* sourceType, + RefType* destType, ResultType* labelType, ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnCastFailV5); + + if (!readVarU32(labelRelativeDepth)) { + return fail("unable to read br_on_cast_fail depth"); + } + + if (!readHeapType(nullable, destType)) { + return false; + } + + ValType castToType(*destType); + return checkBrOnCastFailCommonV5(*labelRelativeDepth, sourceType, castToType, + labelType, values); +} + +template +inline bool OpIter::readBrOnNonStructV5(uint32_t* labelRelativeDepth, + ResultType* labelType, + ValueVector* values) { + MOZ_ASSERT(Classify(op_) == OpKind::BrOnNonStructV5); + + if (!readVarU32(labelRelativeDepth)) { + return fail("unable to read br_on_non_struct depth"); + } + + RefType sourceTypeIgnored; + + // The casted to type is a non-nullable reference to a struct. + ValType castToType(RefType::struct_().asNonNullable()); + + return checkBrOnCastFailCommonV5(*labelRelativeDepth, &sourceTypeIgnored, + castToType, labelType, values); +} + +template +inline bool OpIter::readRefConversion(RefType operandType, + RefType resultType, + Value* operandValue) { + MOZ_ASSERT(Classify(op_) == OpKind::RefConversion); + + StackType actualOperandType; + if (!popWithType(ValType(operandType), operandValue, &actualOperandType)) { + return false; + } + + // The result nullability is the same as the operand nullability + bool outputNullable = actualOperandType.isNullableAsOperand(); + infalliblePush(ValType(resultType.withIsNullable(outputNullable))); + return true; +} + +#endif // ENABLE_WASM_GC + +#ifdef ENABLE_WASM_SIMD + +template +inline bool OpIter::readLaneIndex(uint32_t inputLanes, + uint32_t* laneIndex) { + uint8_t tmp; + if (!readFixedU8(&tmp)) { + return false; // Caller signals error + } + if (tmp >= inputLanes) { + return false; + } + *laneIndex = tmp; + return true; +} + +template +inline bool OpIter::readExtractLane(ValType resultType, + uint32_t inputLanes, + uint32_t* laneIndex, Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::ExtractLane); + + if (!readLaneIndex(inputLanes, laneIndex)) { + return fail("missing or invalid extract_lane lane index"); + } + + if (!popWithType(ValType::V128, input)) { + return false; + } + + infalliblePush(resultType); + + return true; +} + +template +inline bool OpIter::readReplaceLane(ValType operandType, + uint32_t inputLanes, + uint32_t* laneIndex, + Value* baseValue, Value* operand) { + MOZ_ASSERT(Classify(op_) == OpKind::ReplaceLane); + + if (!readLaneIndex(inputLanes, laneIndex)) { + return fail("missing or invalid replace_lane lane index"); + } + + if (!popWithType(operandType, operand)) { + return false; + } + + if (!popWithType(ValType::V128, baseValue)) { + return false; + } + + infalliblePush(ValType::V128); + + return true; +} + +template +inline bool OpIter::readVectorShift(Value* baseValue, Value* shift) { + MOZ_ASSERT(Classify(op_) == OpKind::VectorShift); + + if (!popWithType(ValType::I32, shift)) { + return false; + } + + if (!popWithType(ValType::V128, baseValue)) { + return false; + } + + infalliblePush(ValType::V128); + + return true; +} + +template +inline bool OpIter::readVectorShuffle(Value* v1, Value* v2, + V128* selectMask) { + MOZ_ASSERT(Classify(op_) == OpKind::VectorShuffle); + + for (unsigned char& byte : selectMask->bytes) { + uint8_t tmp; + if (!readFixedU8(&tmp)) { + return fail("unable to read shuffle index"); + } + if (tmp > 31) { + return fail("shuffle index out of range"); + } + byte = tmp; + } + + if (!popWithType(ValType::V128, v2)) { + return false; + } + + if (!popWithType(ValType::V128, v1)) { + return false; + } + + infalliblePush(ValType::V128); + + return true; +} + +template +inline bool OpIter::readV128Const(V128* value) { + MOZ_ASSERT(Classify(op_) == OpKind::V128); + + if (!d_.readV128Const(value)) { + return false; + } + + return push(ValType::V128); +} + +template +inline bool OpIter::readLoadSplat(uint32_t byteSize, + LinearMemoryAddress* addr) { + MOZ_ASSERT(Classify(op_) == OpKind::Load); + + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + infalliblePush(ValType::V128); + + return true; +} + +template +inline bool OpIter::readLoadExtend(LinearMemoryAddress* addr) { + MOZ_ASSERT(Classify(op_) == OpKind::Load); + + if (!readLinearMemoryAddress(/*byteSize=*/8, addr)) { + return false; + } + + infalliblePush(ValType::V128); + + return true; +} + +template +inline bool OpIter::readLoadLane(uint32_t byteSize, + LinearMemoryAddress* addr, + uint32_t* laneIndex, Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::LoadLane); + + if (!popWithType(ValType::V128, input)) { + return false; + } + + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + uint32_t inputLanes = 16 / byteSize; + if (!readLaneIndex(inputLanes, laneIndex)) { + return fail("missing or invalid load_lane lane index"); + } + + infalliblePush(ValType::V128); + + return true; +} + +template +inline bool OpIter::readStoreLane(uint32_t byteSize, + LinearMemoryAddress* addr, + uint32_t* laneIndex, Value* input) { + MOZ_ASSERT(Classify(op_) == OpKind::StoreLane); + + if (!popWithType(ValType::V128, input)) { + return false; + } + + if (!readLinearMemoryAddress(byteSize, addr)) { + return false; + } + + uint32_t inputLanes = 16 / byteSize; + if (!readLaneIndex(inputLanes, laneIndex)) { + return fail("missing or invalid store_lane lane index"); + } + + return true; +} + +#endif // ENABLE_WASM_SIMD + +template +inline bool OpIter::readIntrinsic(const Intrinsic** intrinsic, + ValueVector* params) { + MOZ_ASSERT(Classify(op_) == OpKind::Intrinsic); + + uint32_t id; + if (!d_.readVarU32(&id)) { + return false; + } + + if (id >= uint32_t(IntrinsicId::Limit)) { + return fail("intrinsic index out of range"); + } + + *intrinsic = &Intrinsic::getFromId(IntrinsicId(id)); + + if (!env_.usesMemory()) { + return fail("can't touch memory without memory"); + } + return popWithTypes((*intrinsic)->params, params); +} + +} // namespace wasm +} // namespace js + +static_assert(std::is_trivially_copyable< + js::wasm::TypeAndValueT>::value, + "Must be trivially copyable"); +static_assert(std::is_trivially_destructible< + js::wasm::TypeAndValueT>::value, + "Must be trivially destructible"); + +static_assert(std::is_trivially_copyable< + js::wasm::ControlStackEntry>::value, + "Must be trivially copyable"); +static_assert(std::is_trivially_destructible< + js::wasm::ControlStackEntry>::value, + "Must be trivially destructible"); + +#endif // wasm_op_iter_h diff --git a/js/src/wasm/WasmProcess.cpp b/js/src/wasm/WasmProcess.cpp new file mode 100644 index 0000000000..9b8121ceec --- /dev/null +++ b/js/src/wasm/WasmProcess.cpp @@ -0,0 +1,438 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2017 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmProcess.h" + +#include "mozilla/Attributes.h" +#include "mozilla/BinarySearch.h" +#include "mozilla/ScopeExit.h" + +#include "gc/Memory.h" +#include "threading/ExclusiveData.h" +#include "vm/MutexIDs.h" +#include "vm/Runtime.h" +#include "wasm/WasmBuiltins.h" +#include "wasm/WasmCode.h" +#include "wasm/WasmInstance.h" + +using namespace js; +using namespace wasm; + +using mozilla::BinarySearchIf; + +// Per-process map from values of program-counter (pc) to CodeSegments. +// +// Whenever a new CodeSegment is ready to use, it has to be registered so that +// we can have fast lookups from pc to CodeSegments in numerous places. Since +// wasm compilation may be tiered, and the second tier doesn't have access to +// any JSContext/JS::Compartment/etc lying around, we have to use a process-wide +// map instead. + +using CodeSegmentVector = Vector; + +Atomic wasm::CodeExists(false); + +// Because of profiling, the thread running wasm might need to know to which +// CodeSegment the current PC belongs, during a call to lookup(). A lookup +// is a read-only operation, and we don't want to take a lock then +// (otherwise, we could have a deadlock situation if an async lookup +// happened on a given thread that was holding mutatorsMutex_ while getting +// sampled). Since the writer could be modifying the data that is getting +// looked up, the writer functions use spin-locks to know if there are any +// observers (i.e. calls to lookup()) of the atomic data. + +static Atomic sNumActiveLookups(0); + +class ProcessCodeSegmentMap { + // Since writes (insertions or removals) can happen on any background + // thread at the same time, we need a lock here. + + Mutex mutatorsMutex_ MOZ_UNANNOTATED; + + CodeSegmentVector segments1_; + CodeSegmentVector segments2_; + + // Except during swapAndWait(), there are no lookup() observers of the + // vector pointed to by mutableCodeSegments_ + + CodeSegmentVector* mutableCodeSegments_; + Atomic readonlyCodeSegments_; + + struct CodeSegmentPC { + const void* pc; + explicit CodeSegmentPC(const void* pc) : pc(pc) {} + int operator()(const CodeSegment* cs) const { + if (cs->containsCodePC(pc)) { + return 0; + } + if (pc < cs->base()) { + return -1; + } + return 1; + } + }; + + void swapAndWait() { + // Both vectors are consistent for lookup at this point although their + // contents are different: there is no way for the looked up PC to be + // in the code segment that is getting registered, because the code + // segment is not even fully created yet. + + // If a lookup happens before this instruction, then the + // soon-to-become-former read-only pointer is used during the lookup, + // which is valid. + + mutableCodeSegments_ = const_cast( + readonlyCodeSegments_.exchange(mutableCodeSegments_)); + + // If a lookup happens after this instruction, then the updated vector + // is used, which is valid: + // - in case of insertion, it means the new vector contains more data, + // but it's fine since the code segment is getting registered and thus + // isn't even fully created yet, so the code can't be running. + // - in case of removal, it means the new vector contains one less + // entry, but it's fine since unregistering means the code segment + // isn't used by any live instance anymore, thus PC can't be in the + // to-be-removed code segment's range. + + // A lookup could have happened on any of the two vectors. Wait for + // observers to be done using any vector before mutating. + + while (sNumActiveLookups > 0) { + } + } + + public: + ProcessCodeSegmentMap() + : mutatorsMutex_(mutexid::WasmCodeSegmentMap), + mutableCodeSegments_(&segments1_), + readonlyCodeSegments_(&segments2_) {} + + ~ProcessCodeSegmentMap() { + MOZ_RELEASE_ASSERT(sNumActiveLookups == 0); + MOZ_ASSERT(segments1_.empty()); + MOZ_ASSERT(segments2_.empty()); + segments1_.clearAndFree(); + segments2_.clearAndFree(); + } + + bool insert(const CodeSegment* cs) { + LockGuard lock(mutatorsMutex_); + + size_t index; + MOZ_ALWAYS_FALSE(BinarySearchIf(*mutableCodeSegments_, 0, + mutableCodeSegments_->length(), + CodeSegmentPC(cs->base()), &index)); + + if (!mutableCodeSegments_->insert(mutableCodeSegments_->begin() + index, + cs)) { + return false; + } + + CodeExists = true; + + swapAndWait(); + +#ifdef DEBUG + size_t otherIndex; + MOZ_ALWAYS_FALSE(BinarySearchIf(*mutableCodeSegments_, 0, + mutableCodeSegments_->length(), + CodeSegmentPC(cs->base()), &otherIndex)); + MOZ_ASSERT(index == otherIndex); +#endif + + // Although we could simply revert the insertion in the read-only + // vector, it is simpler to just crash and given that each CodeSegment + // consumes multiple pages, it is unlikely this insert() would OOM in + // practice + AutoEnterOOMUnsafeRegion oom; + if (!mutableCodeSegments_->insert(mutableCodeSegments_->begin() + index, + cs)) { + oom.crash("when inserting a CodeSegment in the process-wide map"); + } + + return true; + } + + void remove(const CodeSegment* cs) { + LockGuard lock(mutatorsMutex_); + + size_t index; + MOZ_ALWAYS_TRUE(BinarySearchIf(*mutableCodeSegments_, 0, + mutableCodeSegments_->length(), + CodeSegmentPC(cs->base()), &index)); + + mutableCodeSegments_->erase(mutableCodeSegments_->begin() + index); + + if (!mutableCodeSegments_->length()) { + CodeExists = false; + } + + swapAndWait(); + +#ifdef DEBUG + size_t otherIndex; + MOZ_ALWAYS_TRUE(BinarySearchIf(*mutableCodeSegments_, 0, + mutableCodeSegments_->length(), + CodeSegmentPC(cs->base()), &otherIndex)); + MOZ_ASSERT(index == otherIndex); +#endif + + mutableCodeSegments_->erase(mutableCodeSegments_->begin() + index); + } + + const CodeSegment* lookup(const void* pc) { + const CodeSegmentVector* readonly = readonlyCodeSegments_; + + size_t index; + if (!BinarySearchIf(*readonly, 0, readonly->length(), CodeSegmentPC(pc), + &index)) { + return nullptr; + } + + // It is fine returning a raw CodeSegment*, because we assume we are + // looking up a live PC in code which is on the stack, keeping the + // CodeSegment alive. + + return (*readonly)[index]; + } +}; + +// This field is only atomic to handle buggy scenarios where we crash during +// startup or shutdown and thus racily perform wasm::LookupCodeSegment() from +// the crashing thread. + +static Atomic sProcessCodeSegmentMap(nullptr); + +bool wasm::RegisterCodeSegment(const CodeSegment* cs) { + MOZ_ASSERT(cs->codeTier().code().initialized()); + + // This function cannot race with startup/shutdown. + ProcessCodeSegmentMap* map = sProcessCodeSegmentMap; + MOZ_RELEASE_ASSERT(map); + return map->insert(cs); +} + +void wasm::UnregisterCodeSegment(const CodeSegment* cs) { + // This function cannot race with startup/shutdown. + ProcessCodeSegmentMap* map = sProcessCodeSegmentMap; + MOZ_RELEASE_ASSERT(map); + map->remove(cs); +} + +const CodeSegment* wasm::LookupCodeSegment( + const void* pc, const CodeRange** codeRange /*= nullptr */) { + // Since wasm::LookupCodeSegment() can race with wasm::ShutDown(), we must + // additionally keep sNumActiveLookups above zero for the duration we're + // using the ProcessCodeSegmentMap. wasm::ShutDown() spin-waits on + // sNumActiveLookups getting to zero. + + auto decObserver = mozilla::MakeScopeExit([&] { + MOZ_ASSERT(sNumActiveLookups > 0); + sNumActiveLookups--; + }); + sNumActiveLookups++; + + ProcessCodeSegmentMap* map = sProcessCodeSegmentMap; + if (!map) { + return nullptr; + } + + if (const CodeSegment* found = map->lookup(pc)) { + if (codeRange) { + *codeRange = found->isModule() ? found->asModule()->lookupRange(pc) + : found->asLazyStub()->lookupRange(pc); + } + return found; + } + + if (codeRange) { + *codeRange = nullptr; + } + + return nullptr; +} + +const Code* wasm::LookupCode(const void* pc, + const CodeRange** codeRange /* = nullptr */) { + const CodeSegment* found = LookupCodeSegment(pc, codeRange); + MOZ_ASSERT_IF(!found && codeRange, !*codeRange); + return found ? &found->code() : nullptr; +} + +bool wasm::InCompiledCode(void* pc) { + if (LookupCodeSegment(pc)) { + return true; + } + + const CodeRange* codeRange; + uint8_t* codeBase; + return LookupBuiltinThunk(pc, &codeRange, &codeBase); +} + +/** + * ReadLockFlag maintains a flag that can be mutated multiple times before it + * is read, at which point it maintains the same value. + */ +class ReadLockFlag { + private: + bool enabled_; + bool read_; + + public: + ReadLockFlag() : enabled_(false), read_(false) {} + + bool get() { + read_ = true; + return enabled_; + } + + bool set(bool enabled) { + if (read_) { + return false; + } + enabled_ = enabled; + return true; + } +}; + +#ifdef WASM_SUPPORTS_HUGE_MEMORY +/* + * Some 64 bit systems greatly limit the range of available virtual memory. We + * require about 6GiB for each wasm huge memory, which can exhaust the address + * spaces of these systems quickly. In order to avoid this, we only enable huge + * memory if we observe a large enough address space. + * + * This number is conservatively chosen to continue using huge memory on our + * smallest address space system, Android on ARM64 (39 bits), along with a bit + * for error in detecting the address space limit. + */ +static const size_t MinAddressBitsForHugeMemory = 38; + +/* + * In addition to the above, some systems impose an independent limit on the + * amount of virtual memory that may be used. + */ +static const size_t MinVirtualMemoryLimitForHugeMemory = + size_t(1) << MinAddressBitsForHugeMemory; +#endif + +ExclusiveData sHugeMemoryEnabled32( + mutexid::WasmHugeMemoryEnabled); +ExclusiveData sHugeMemoryEnabled64( + mutexid::WasmHugeMemoryEnabled); + +static MOZ_NEVER_INLINE bool IsHugeMemoryEnabledHelper32() { + auto state = sHugeMemoryEnabled32.lock(); + return state->get(); +} + +static MOZ_NEVER_INLINE bool IsHugeMemoryEnabledHelper64() { + auto state = sHugeMemoryEnabled64.lock(); + return state->get(); +} + +bool wasm::IsHugeMemoryEnabled(wasm::IndexType t) { + if (t == IndexType::I32) { + static bool enabled32 = IsHugeMemoryEnabledHelper32(); + return enabled32; + } + static bool enabled64 = IsHugeMemoryEnabledHelper64(); + return enabled64; +} + +bool wasm::DisableHugeMemory() { + bool ok = true; + { + auto state = sHugeMemoryEnabled64.lock(); + ok = ok && state->set(false); + } + { + auto state = sHugeMemoryEnabled32.lock(); + ok = ok && state->set(false); + } + return ok; +} + +void ConfigureHugeMemory() { +#ifdef WASM_SUPPORTS_HUGE_MEMORY + bool ok = true; + + { + // Currently no huge memory for IndexType::I64, so always set to false. + auto state = sHugeMemoryEnabled64.lock(); + ok = ok && state->set(false); + } + + if (gc::SystemAddressBits() < MinAddressBitsForHugeMemory) { + return; + } + + if (gc::VirtualMemoryLimit() != size_t(-1) && + gc::VirtualMemoryLimit() < MinVirtualMemoryLimitForHugeMemory) { + return; + } + + { + auto state = sHugeMemoryEnabled32.lock(); + ok = ok && state->set(true); + } + + MOZ_RELEASE_ASSERT(ok); +#endif +} + +bool wasm::Init() { + MOZ_RELEASE_ASSERT(!sProcessCodeSegmentMap); + + uintptr_t pageSize = gc::SystemPageSize(); + MOZ_RELEASE_ASSERT(wasm::NullPtrGuardSize <= pageSize); + + ConfigureHugeMemory(); + + AutoEnterOOMUnsafeRegion oomUnsafe; + ProcessCodeSegmentMap* map = js_new(); + if (!map) { + oomUnsafe.crash("js::wasm::Init"); + } + + sProcessCodeSegmentMap = map; + return true; +} + +void wasm::ShutDown() { + // If there are live runtimes then we are already pretty much leaking the + // world, so to avoid spurious assertions (which are valid and valuable when + // there are not live JSRuntimes), don't bother releasing anything here. + if (JSRuntime::hasLiveRuntimes()) { + return; + } + + PurgeCanonicalTypes(); + + // After signalling shutdown by clearing sProcessCodeSegmentMap, wait for + // concurrent wasm::LookupCodeSegment()s to finish. + ProcessCodeSegmentMap* map = sProcessCodeSegmentMap; + MOZ_RELEASE_ASSERT(map); + sProcessCodeSegmentMap = nullptr; + while (sNumActiveLookups > 0) { + } + + ReleaseBuiltinThunks(); + js_delete(map); +} diff --git a/js/src/wasm/WasmProcess.h b/js/src/wasm/WasmProcess.h new file mode 100644 index 0000000000..89a0b733c9 --- /dev/null +++ b/js/src/wasm/WasmProcess.h @@ -0,0 +1,76 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2017 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_process_h +#define wasm_process_h + +#include "mozilla/Atomics.h" + +#include "wasm/WasmMemory.h" + +namespace js { +namespace wasm { + +class Code; +class CodeRange; +class CodeSegment; + +// These methods return the wasm::CodeSegment (resp. wasm::Code) containing +// the given pc, if any exist in the process. These methods do not take a lock, +// and thus are safe to use in a profiling context. + +const CodeSegment* LookupCodeSegment(const void* pc, + const CodeRange** codeRange = nullptr); + +const Code* LookupCode(const void* pc, const CodeRange** codeRange = nullptr); + +// Return whether the given PC is in any type of wasm code (module or builtin). + +bool InCompiledCode(void* pc); + +// A bool member that can be used as a very fast lookup to know if there is any +// code segment at all. + +extern mozilla::Atomic CodeExists; + +// These methods allow to (un)register CodeSegments so they can be looked up +// via pc in the methods described above. + +bool RegisterCodeSegment(const CodeSegment* cs); + +void UnregisterCodeSegment(const CodeSegment* cs); + +// Whether this process is configured to use huge memory or not. Note that this +// is not precise enough to tell whether a particular memory uses huge memory, +// there are additional conditions for that. + +bool IsHugeMemoryEnabled(IndexType t); + +[[nodiscard]] bool DisableHugeMemory(); + +// Called once before/after the last VM execution which could execute or compile +// wasm. + +bool Init(); + +void ShutDown(); + +} // namespace wasm +} // namespace js + +#endif // wasm_process_h diff --git a/js/src/wasm/WasmRealm.cpp b/js/src/wasm/WasmRealm.cpp new file mode 100644 index 0000000000..31849e15a2 --- /dev/null +++ b/js/src/wasm/WasmRealm.cpp @@ -0,0 +1,150 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmRealm.h" + +#include "vm/Realm.h" +#include "wasm/WasmDebug.h" +#include "wasm/WasmInstance.h" + +#include "debugger/DebugAPI-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace wasm; + +wasm::Realm::Realm(JSRuntime* rt) : runtime_(rt) {} + +wasm::Realm::~Realm() { MOZ_ASSERT(instances_.empty()); } + +struct InstanceComparator { + const Instance& target; + explicit InstanceComparator(const Instance& target) : target(target) {} + + int operator()(const Instance* instance) const { + if (instance == &target) { + return 0; + } + + // Instances can share code, so the segments can be equal (though they + // can't partially overlap). If the codeBases are equal, we sort by + // Instance address. Thus a Code may map to many instances. + + // Compare by the first tier, always. + + Tier instanceTier = instance->code().stableTier(); + Tier targetTier = target.code().stableTier(); + + if (instance->codeBase(instanceTier) == target.codeBase(targetTier)) { + return instance < &target ? -1 : 1; + } + + return target.codeBase(targetTier) < instance->codeBase(instanceTier) ? -1 + : 1; + } +}; + +bool wasm::Realm::registerInstance(JSContext* cx, + Handle instanceObj) { + MOZ_ASSERT(runtime_ == cx->runtime()); + + Instance& instance = instanceObj->instance(); + MOZ_ASSERT(this == &instance.realm()->wasm); + + instance.ensureProfilingLabels(cx->runtime()->geckoProfiler().enabled()); + + if (instance.debugEnabled() && + instance.realm()->debuggerObservesAllExecution()) { + instance.debug().ensureEnterFrameTrapsState(cx, &instance, true); + } + + { + if (!instances_.reserve(instances_.length() + 1)) { + return false; + } + + auto runtimeInstances = cx->runtime()->wasmInstances.lock(); + if (!runtimeInstances->reserve(runtimeInstances->length() + 1)) { + return false; + } + + // To avoid implementing rollback, do not fail after mutations start. + + InstanceComparator cmp(instance); + size_t index; + + // The following section is not unsafe, but simulated OOM do not consider + // the fact that these insert calls are guarded by the previous reserve + // calls. + AutoEnterOOMUnsafeRegion oomUnsafe; + (void)oomUnsafe; + + MOZ_ALWAYS_FALSE( + BinarySearchIf(instances_, 0, instances_.length(), cmp, &index)); + MOZ_ALWAYS_TRUE(instances_.insert(instances_.begin() + index, &instance)); + + MOZ_ALWAYS_FALSE(BinarySearchIf(runtimeInstances.get(), 0, + runtimeInstances->length(), cmp, &index)); + MOZ_ALWAYS_TRUE( + runtimeInstances->insert(runtimeInstances->begin() + index, &instance)); + } + + // Notify the debugger after wasmInstances is unlocked. + DebugAPI::onNewWasmInstance(cx, instanceObj); + return true; +} + +void wasm::Realm::unregisterInstance(Instance& instance) { + InstanceComparator cmp(instance); + size_t index; + + if (BinarySearchIf(instances_, 0, instances_.length(), cmp, &index)) { + instances_.erase(instances_.begin() + index); + } + + auto runtimeInstances = runtime_->wasmInstances.lock(); + if (BinarySearchIf(runtimeInstances.get(), 0, runtimeInstances->length(), cmp, + &index)) { + runtimeInstances->erase(runtimeInstances->begin() + index); + } +} + +void wasm::Realm::ensureProfilingLabels(bool profilingEnabled) { + for (Instance* instance : instances_) { + instance->ensureProfilingLabels(profilingEnabled); + } +} + +void wasm::Realm::addSizeOfExcludingThis(MallocSizeOf mallocSizeOf, + size_t* realmTables) { + *realmTables += instances_.sizeOfExcludingThis(mallocSizeOf); +} + +void wasm::InterruptRunningCode(JSContext* cx) { + auto runtimeInstances = cx->runtime()->wasmInstances.lock(); + for (Instance* instance : runtimeInstances.get()) { + instance->setInterrupt(); + } +} + +void wasm::ResetInterruptState(JSContext* cx) { + auto runtimeInstances = cx->runtime()->wasmInstances.lock(); + for (Instance* instance : runtimeInstances.get()) { + instance->resetInterrupt(cx); + } +} diff --git a/js/src/wasm/WasmRealm.h b/js/src/wasm/WasmRealm.h new file mode 100644 index 0000000000..4216d4d7b5 --- /dev/null +++ b/js/src/wasm/WasmRealm.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_realm_h +#define wasm_realm_h + +#include "wasm/WasmTypeDecls.h" + +namespace js { +namespace wasm { + +// wasm::Realm lives in JS::Realm and contains the wasm-related per-realm state. +// wasm::Realm tracks every live instance in the realm and must be notified, via +// registerInstance(), of any new WasmInstanceObject. + +class Realm { + JSRuntime* runtime_; + InstanceVector instances_; + + public: + explicit Realm(JSRuntime* rt); + ~Realm(); + + // Before a WasmInstanceObject can be considered fully constructed and + // valid, it must be registered with the Realm. If this method fails, + // an error has been reported and the instance object must be abandoned. + // After a successful registration, an Instance must call + // unregisterInstance() before being destroyed. + + bool registerInstance(JSContext* cx, Handle instanceObj); + void unregisterInstance(Instance& instance); + + // Return a vector of all live instances in the realm. The lifetime of + // these Instances is determined by their owning WasmInstanceObject. + // Note that accessing instances()[i]->object() triggers a read barrier + // since instances() is effectively a weak list. + + const InstanceVector& instances() const { return instances_; } + + // Ensure all Instances in this Realm have profiling labels created. + + void ensureProfilingLabels(bool profilingEnabled); + + // about:memory reporting + + void addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf, + size_t* realmTables); +}; + +// Interrupt all running wasm Instances that have been registered with +// wasm::Realms in the given JSContext. + +extern void InterruptRunningCode(JSContext* cx); + +// After a wasm Instance sees an interrupt request and calls +// CheckForInterrupt(), it should call RunningCodeInterrupted() to clear the +// interrupt request for all wasm Instances to avoid spurious trapping. + +void ResetInterruptState(JSContext* cx); + +} // namespace wasm +} // namespace js + +#endif // wasm_realm_h diff --git a/js/src/wasm/WasmSerialize.cpp b/js/src/wasm/WasmSerialize.cpp new file mode 100644 index 0000000000..7a20f884e2 --- /dev/null +++ b/js/src/wasm/WasmSerialize.cpp @@ -0,0 +1,1230 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "wasm/WasmSerialize.h" + +#include "mozilla/EnumeratedRange.h" +#include "mozilla/Maybe.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Vector.h" + +#include +#include +#include +#include + +#include "js/StreamConsumer.h" +#include "wasm/WasmCode.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmGC.h" +#include "wasm/WasmInitExpr.h" +#include "wasm/WasmModule.h" +#include "wasm/WasmModuleTypes.h" +#include "wasm/WasmTypeDef.h" +#include "wasm/WasmValType.h" +#include "wasm/WasmValue.h" + +using namespace js; +using namespace js::wasm; + +using mozilla::Err; +using mozilla::Ok; + +namespace js { +namespace wasm { + +// The following assert is used as a tripwire for for new fields being added to +// types. If this assertion is broken by your code, verify the serialization +// code is correct, and then update the assertion. +// +// We only check serialized type sizes on one 'golden' platform. The platform +// is arbitrary, but must remain consistent. The platform should be as specific +// as possible so these assertions don't erroneously fire. Checkout the +// definition of ENABLE_WASM_VERIFY_SERIALIZATION_FOR_SIZE in js/moz.configure +// for the current platform. +// +// If this mechanism becomes a hassle, we can investigate other methods of +// achieving the same goal. +#if defined(ENABLE_WASM_VERIFY_SERIALIZATION_FOR_SIZE) + +template +struct Tripwire { + // The following will print a compile error that contains the size of type, + // and can be used for updating the assertion to the correct size. + static char (*_Error)[sizeof(T)] = 1; + + // We must reference the bad declaration to work around SFINAE. + static const bool Value = !_Error; +}; + +template +struct Tripwire { + static const bool Value = true; +}; + +# define WASM_VERIFY_SERIALIZATION_FOR_SIZE(Type, Size) \ + static_assert(Tripwire::Value); + +#else +# define WASM_VERIFY_SERIALIZATION_FOR_SIZE(Type, Size) static_assert(true); +#endif + +// A pointer is not cacheable POD +static_assert(!is_cacheable_pod); + +// A non-fixed sized array is not cacheable POD +static_assert(!is_cacheable_pod); + +// Cacheable POD is not inherited +struct TestPodBase {}; +WASM_DECLARE_CACHEABLE_POD(TestPodBase); +struct InheritTestPodBase : public TestPodBase {}; +static_assert(is_cacheable_pod && + !is_cacheable_pod); + +// Coding functions for containers and smart pointers need to know which code +// function to apply to the inner value. 'CodeFunc' is the common signature to +// be used for this. +template +using CodeFunc = CoderResult (*)(Coder&, CoderArg); + +// Some functions are generic for MODE_SIZE and MODE_ENCODE, but not +// MODE_DECODE. This assert is used to ensure that the right function overload +// is chosen in these cases. +#define STATIC_ASSERT_ENCODING_OR_SIZING \ + static_assert(mode == MODE_ENCODE || mode == MODE_SIZE, "wrong overload"); + +CoderResult Coder::writeBytes(const void* unusedSrc, size_t length) { + size_ += length; + if (!size_.isValid()) { + return Err(OutOfMemory()); + } + return Ok(); +} + +CoderResult Coder::writeBytes(const void* src, size_t length) { + MOZ_RELEASE_ASSERT(buffer_ + length <= end_); + memcpy(buffer_, src, length); + buffer_ += length; + return Ok(); +} + +CoderResult Coder::readBytes(void* dest, size_t length) { + MOZ_RELEASE_ASSERT(buffer_ + length <= end_); + memcpy(dest, buffer_, length); + buffer_ += length; + return Ok(); +} + +// Cacheable POD coding functions + +template , bool> = true> +CoderResult CodePod(Coder& coder, T* item) { + return coder.readBytes((void*)item, sizeof(T)); +} + +template , bool> = true> +CoderResult CodePod(Coder& coder, const T* item) { + STATIC_ASSERT_ENCODING_OR_SIZING; + return coder.writeBytes((const void*)item, sizeof(T)); +} + +// "Magic Marker". Use to sanity check the serialization process. + +enum class Marker : uint32_t { + LinkData = 0x49102278, + Imports, + Exports, + DataSegments, + ElemSegments, + CustomSections, + Code, + Metadata, + MetadataTier, + CodeTier, + ModuleSegment, +}; + +template +CoderResult Magic(Coder& coder, Marker item) { + if constexpr (mode == MODE_DECODE) { + // Assert the specified marker is in the binary + Marker decoded; + MOZ_TRY(CodePod(coder, &decoded)); + MOZ_RELEASE_ASSERT(decoded == item); + return Ok(); + } else { + // Encode the specified marker in the binary + return CodePod(coder, &item); + } +} + +// mozilla::Maybe coding functions +// +// These functions will only code the inner value if Maybe.isSome(). The +// coding function to use for the inner value is specified by a template +// parameter. + +template CodeT> +CoderResult CodeMaybe(Coder& coder, Maybe* item) { + // Decode 'isSome()' + uint8_t isSome; + MOZ_TRY(CodePod(coder, &isSome)); + + if (isSome == 1) { + // Initialize to Some with default constructor + item->emplace(); + // Code the inner type + MOZ_TRY(CodeT(coder, item->ptr())); + } else { + // Initialize to nothing + *item = mozilla::Nothing(); + } + return Ok(); +} + +template CodeT> +CoderResult CodeMaybe(Coder& coder, const Maybe* item) { + STATIC_ASSERT_ENCODING_OR_SIZING; + + // Encode or size 'isSome()' + const uint8_t isSome = item->isSome() ? 1 : 0; + MOZ_TRY(CodePod(coder, &isSome)); + + if (item->isSome()) { + // Encode or size the inner value + MOZ_TRY(CodeT(coder, item->ptr())); + } + return Ok(); +} + +// Cacheable POD mozilla::Vector coding functions +// +// These functions are only available if the element type is cacheable POD. In +// this case, the whole contents of the vector are copied directly to/from the +// buffer. + +template , bool> = true> +CoderResult CodePodVector(Coder& coder, + Vector* item) { + // Decode the length + size_t length; + MOZ_TRY(CodePod(coder, &length)); + + // Prepare to copy into the vector + if (!item->initLengthUninitialized(length)) { + return Err(OutOfMemory()); + } + + // Copy directly from the buffer to the vector + const size_t byteLength = length * sizeof(T); + return coder.readBytes((void*)item->begin(), byteLength); +} + +template , bool> = true> +CoderResult CodePodVector(Coder& coder, + const Vector* item) { + STATIC_ASSERT_ENCODING_OR_SIZING; + + // Encode the length + const size_t length = item->length(); + MOZ_TRY(CodePod(coder, &length)); + + // Copy directly from the vector to the buffer + const size_t byteLength = length * sizeof(T); + return coder.writeBytes((const void*)item->begin(), byteLength); +} + +// Non-cacheable-POD mozilla::Vector coding functions +// +// These functions implement the general case of coding a vector of some type. +// The coding function to use on the vector elements is provided through a +// template parameter. + +template CodeT, size_t N, + typename std::enable_if_t, bool> = true> +CoderResult CodeVector(Coder& coder, + Vector* item) { + // Decode the length + size_t length; + MOZ_TRY(CodePod(coder, &length)); + + // Attempt to grow the buffer to length, this will default initialize each + // element + if (!item->resize(length)) { + return Err(OutOfMemory()); + } + + // Decode each child element from the buffer + for (auto iter = item->begin(); iter != item->end(); iter++) { + MOZ_TRY(CodeT(coder, iter)); + } + return Ok(); +} + +template CodeT, size_t N, + typename std::enable_if_t, bool> = true> +CoderResult CodeVector(Coder& coder, + const Vector* item) { + STATIC_ASSERT_ENCODING_OR_SIZING; + + // Encode the length + const size_t length = item->length(); + MOZ_TRY(CodePod(coder, &length)); + + // Encode each child element + for (auto iter = item->begin(); iter != item->end(); iter++) { + MOZ_TRY(CodeT(coder, iter)); + } + return Ok(); +} + +// This function implements encoding and decoding of RefPtr. A coding +// function is provided for the inner value through a template parameter. +// +// The special handling of const qualification allows a RefPtr to be +// decoded correctly. +template > CodeT> +CoderResult CodeRefPtr(Coder& coder, CoderArg> item) { + if constexpr (mode == MODE_DECODE) { + // The RefPtr should not be initialized yet + MOZ_ASSERT(!item->get()); + + // Allocate and default construct the inner type + auto* allocated = js_new>(); + if (!allocated) { + return Err(OutOfMemory()); + } + + // Initialize the RefPtr + *item = allocated; + + // Decode the inner type + MOZ_TRY(CodeT(coder, allocated)); + return Ok(); + } else { + // Encode the inner type + return CodeT(coder, item->get()); + } +} + +// This function implements encoding and decoding of UniquePtr. +// A coding function is provided for the inner value as a function parameter. +// The coding function may accept additional parameters to forward to the inner +// coding function. +// +// The inner coding function is provided as an argument instead of a template +// parameter in order to make the variable arguments template parameter +// simpler. +template +CoderResult CodeUniquePtr(Coder& coder, CoderArg> item, + CodeTFunctor innerCode, Args&&... args) { + if constexpr (mode == MODE_DECODE) { + // The UniquePtr should not be initialized yet + MOZ_ASSERT(!item->get()); + + // Allocate and default construct the inner type + auto allocated = js::MakeUnique>(); + if (!allocated.get()) { + return Err(OutOfMemory()); + } + + // Decode the inner type + MOZ_TRY(innerCode(coder, allocated.get(), std::forward(args)...)); + + // Initialize the UniquePtr + *item = std::move(allocated); + return Ok(); + } else { + // Encode the inner type + return innerCode(coder, item->get(), std::forward(args)...); + } +} + +// CacheableChars coding functions + +static size_t StringLengthWithNullChar(const char* chars) { + return chars ? strlen(chars) + 1 : 0; +} + +CoderResult CodeCacheableChars(Coder& coder, + CacheableChars* item) { + uint32_t lengthWithNullChar; + MOZ_TRY(CodePod(coder, &lengthWithNullChar)); + + // Decode the bytes, if any + if (lengthWithNullChar) { + item->reset(js_pod_malloc(lengthWithNullChar)); + if (!item->get()) { + return Err(OutOfMemory()); + } + return coder.readBytes((char*)item->get(), lengthWithNullChar); + } + + // If there were no bytes to write, the string should be null + MOZ_ASSERT(!item->get()); + return Ok(); +} + +template +CoderResult CodeCacheableChars(Coder& coder, const CacheableChars* item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::CacheableChars, 8); + STATIC_ASSERT_ENCODING_OR_SIZING; + + // Encode the length + const uint32_t lengthWithNullChar = StringLengthWithNullChar(item->get()); + MOZ_TRY(CodePod(coder, &lengthWithNullChar)); + + // Write the bytes, if any + if (lengthWithNullChar) { + return coder.writeBytes((const void*)item->get(), lengthWithNullChar); + } + + // If there were no bytes to write, the string should be null + MOZ_ASSERT(!item->get()); + return Ok(); +} + +// Code a CacheableName + +template +CoderResult CodeCacheableName(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::CacheableName, 40); + MOZ_TRY(CodePodVector(coder, &item->bytes_)); + return Ok(); +} + +// Code a ShareableBytes. This function only needs to forward to the inner +// bytes vector. +template +CoderResult CodeShareableBytes(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::ShareableBytes, 48); + return CodePodVector(coder, &item->bytes); +} + +// WasmValType.h + +/* static */ +SerializableTypeCode SerializableTypeCode::serialize(PackedTypeCode ptc, + const TypeContext& types) { + SerializableTypeCode stc = {}; + stc.typeCode = PackedRepr(ptc.typeCode()); + stc.typeIndex = ptc.typeDef() ? types.indexOf(*ptc.typeDef()) + : SerializableTypeCode::NoTypeIndex; + stc.nullable = ptc.isNullable(); + return stc; +} + +PackedTypeCode SerializableTypeCode::deserialize(const TypeContext& types) { + if (typeIndex == SerializableTypeCode::NoTypeIndex) { + return PackedTypeCode::pack(TypeCode(typeCode), nullable); + } + const TypeDef* typeDef = &types.type(typeIndex); + return PackedTypeCode::pack(TypeCode(typeCode), typeDef, nullable); +} + +template +CoderResult CodePackedTypeCode(Coder& coder, + CoderArg item) { + if constexpr (mode == MODE_DECODE) { + SerializableTypeCode stc; + MOZ_TRY(CodePod(coder, &stc)); + *item = stc.deserialize(*coder.types_); + return Ok(); + } else if constexpr (mode == MODE_SIZE) { + return coder.writeBytes(nullptr, sizeof(SerializableTypeCode)); + } else { + SerializableTypeCode stc = + SerializableTypeCode::serialize(*item, *coder.types_); + return CodePod(coder, &stc); + } +} + +template +CoderResult CodeValType(Coder& coder, CoderArg item) { + return CodePackedTypeCode(coder, item->addressOfPacked()); +} + +template +CoderResult CodeFieldType(Coder& coder, CoderArg item) { + return CodePackedTypeCode(coder, item->addressOfPacked()); +} + +template +CoderResult CodeRefType(Coder& coder, CoderArg item) { + return CodePackedTypeCode(coder, item->addressOfPacked()); +} + +// WasmValue.h + +template +CoderResult CodeLitVal(Coder& coder, CoderArg item) { + MOZ_TRY(CodeValType(coder, &item->type_)); + MOZ_TRY(CodePod(coder, &item->cell_)); + return Ok(); +} + +// WasmInitExpr.h + +template +CoderResult CodeInitExpr(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::InitExpr, 80); + MOZ_TRY(CodePod(coder, &item->kind_)); + MOZ_TRY(CodeValType(coder, &item->type_)); + switch (item->kind_) { + case InitExprKind::Literal: + MOZ_TRY(CodeLitVal(coder, &item->literal_)); + break; + case InitExprKind::Variable: + MOZ_TRY(CodePodVector(coder, &item->bytecode_)); + break; + default: + MOZ_CRASH(); + } + return Ok(); +} + +// WasmTypeDef.h + +template +CoderResult CodeFuncType(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::FuncType, 344); + MOZ_TRY((CodeVector>(coder, &item->args_))); + MOZ_TRY( + (CodeVector>(coder, &item->results_))); + MOZ_TRY(CodePod(coder, &item->immediateTypeId_)); + return Ok(); +} + +template +CoderResult CodeStructField(Coder& coder, + CoderArg item) { + MOZ_TRY(CodeFieldType(coder, &item->type)); + MOZ_TRY(CodePod(coder, &item->offset)); + MOZ_TRY(CodePod(coder, &item->isMutable)); + return Ok(); +} + +template +CoderResult CodeStructType(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::StructType, 136); + MOZ_TRY((CodeVector>( + coder, &item->fields_))); + MOZ_TRY(CodePod(coder, &item->size_)); + return Ok(); +} + +template +CoderResult CodeArrayType(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::ArrayType, 16); + MOZ_TRY(CodeFieldType(coder, &item->elementType_)); + MOZ_TRY(CodePod(coder, &item->isMutable_)); + return Ok(); +} + +template +CoderResult CodeTypeDef(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::TypeDef, 376); + // TypeDef is a tagged union that begins with kind = None. This implies that + // we must manually initialize the variant that we decode. + if constexpr (mode == MODE_DECODE) { + MOZ_RELEASE_ASSERT(item->kind_ == TypeDefKind::None); + } + MOZ_TRY(CodePod(coder, &item->kind_)); + switch (item->kind_) { + case TypeDefKind::Struct: { + if constexpr (mode == MODE_DECODE) { + new (&item->structType_) StructType(); + } + MOZ_TRY(CodeStructType(coder, &item->structType_)); + break; + } + case TypeDefKind::Func: { + if constexpr (mode == MODE_DECODE) { + new (&item->funcType_) FuncType(); + } + MOZ_TRY(CodeFuncType(coder, &item->funcType_)); + break; + } + case TypeDefKind::Array: { + if constexpr (mode == MODE_DECODE) { + new (&item->arrayType_) ArrayType(); + } + MOZ_TRY(CodeArrayType(coder, &item->arrayType_)); + break; + } + case TypeDefKind::None: { + break; + } + default: + MOZ_ASSERT_UNREACHABLE(); + } + return Ok(); +} + +template +CoderResult CodeTypeContext(Coder& coder, + CoderArg item) { + if constexpr (mode == MODE_DECODE) { + // Decoding type definitions needs to reference the type context of the + // module + MOZ_ASSERT(!coder.types_); + coder.types_ = item; + + // Decode the number of recursion groups in the module + uint32_t numRecGroups; + MOZ_TRY(CodePod(coder, &numRecGroups)); + + // Decode each recursion group + for (uint32_t recGroupIndex = 0; recGroupIndex < numRecGroups; + recGroupIndex++) { + // Decode the number of types in the recursion group + uint32_t numTypes; + MOZ_TRY(CodePod(coder, &numTypes)); + + MutableRecGroup recGroup = item->startRecGroup(numTypes); + if (!recGroup) { + return Err(OutOfMemory()); + } + + // Decode the type definitions + for (uint32_t groupTypeIndex = 0; groupTypeIndex < numTypes; + groupTypeIndex++) { + MOZ_TRY(CodeTypeDef(coder, &recGroup->type(groupTypeIndex))); + } + + // Finish the recursion group + if (!item->endRecGroup()) { + return Err(OutOfMemory()); + } + } + } else { + // Encode the number of recursion groups in the module + uint32_t numRecGroups = item->groups().length(); + MOZ_TRY(CodePod(coder, &numRecGroups)); + + // Encode each recursion group + for (uint32_t groupIndex = 0; groupIndex < numRecGroups; groupIndex++) { + SharedRecGroup group = item->groups()[groupIndex]; + + // Encode the number of types in the recursion group + uint32_t numTypes = group->numTypes(); + MOZ_TRY(CodePod(coder, &numTypes)); + + // Encode the type definitions + for (uint32_t i = 0; i < numTypes; i++) { + MOZ_TRY(CodeTypeDef(coder, &group->type(i))); + } + } + } + return Ok(); +} + +// WasmModuleTypes.h + +template +CoderResult CodeImport(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Import, 88); + MOZ_TRY(CodeCacheableName(coder, &item->module)); + MOZ_TRY(CodeCacheableName(coder, &item->field)); + MOZ_TRY(CodePod(coder, &item->kind)); + return Ok(); +} + +template +CoderResult CodeExport(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Export, 48); + MOZ_TRY(CodeCacheableName(coder, &item->fieldName_)); + MOZ_TRY(CodePod(coder, &item->pod)); + return Ok(); +} + +template +CoderResult CodeGlobalDesc(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::GlobalDesc, 104); + MOZ_TRY(CodePod(coder, &item->kind_)); + MOZ_TRY(CodeInitExpr(coder, &item->initial_)); + MOZ_TRY(CodePod(coder, &item->offset_)); + MOZ_TRY(CodePod(coder, &item->isMutable_)); + MOZ_TRY(CodePod(coder, &item->isWasm_)); + MOZ_TRY(CodePod(coder, &item->isExport_)); + MOZ_TRY(CodePod(coder, &item->importIndex_)); + return Ok(); +} + +template +CoderResult CodeTagType(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::TagType, 232); + MOZ_TRY( + (CodeVector>(coder, &item->argTypes_))); + MOZ_TRY(CodePodVector(coder, &item->argOffsets_)); + MOZ_TRY(CodePod(coder, &item->size_)); + return Ok(); +} + +template +CoderResult CodeTagDesc(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::TagDesc, 24); + MOZ_TRY(CodePod(coder, &item->kind)); + MOZ_TRY(( + CodeRefPtr>(coder, &item->type))); + MOZ_TRY(CodePod(coder, &item->isExport)); + return Ok(); +} + +template +CoderResult CodeElemSegment(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::ElemSegment, 184); + MOZ_TRY(CodePod(coder, &item->kind)); + MOZ_TRY(CodePod(coder, &item->tableIndex)); + MOZ_TRY(CodeRefType(coder, &item->elemType)); + MOZ_TRY((CodeMaybe>( + coder, &item->offsetIfActive))); + MOZ_TRY(CodePodVector(coder, &item->elemFuncIndices)); + return Ok(); +} + +template +CoderResult CodeDataSegment(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::DataSegment, 136); + MOZ_TRY((CodeMaybe>( + coder, &item->offsetIfActive))); + MOZ_TRY(CodePodVector(coder, &item->bytes)); + return Ok(); +} + +template +CoderResult CodeCustomSection(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::CustomSection, 48); + MOZ_TRY(CodePodVector(coder, &item->name)); + MOZ_TRY((CodeRefPtr>( + coder, &item->payload))); + return Ok(); +} + +template +CoderResult CodeTableDesc(Coder& coder, CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::TableDesc, 112); + MOZ_TRY(CodeRefType(coder, &item->elemType)); + MOZ_TRY(CodePod(coder, &item->isImported)); + MOZ_TRY(CodePod(coder, &item->isExported)); + MOZ_TRY(CodePod(coder, &item->isAsmJS)); + MOZ_TRY(CodePod(coder, &item->initialLength)); + MOZ_TRY(CodePod(coder, &item->maximumLength)); + MOZ_TRY( + (CodeMaybe>(coder, &item->initExpr))); + return Ok(); +} + +// WasmCodegenTypes.h + +template +CoderResult CodeTrapSiteVectorArray(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::TrapSiteVectorArray, 520); + for (Trap trap : mozilla::MakeEnumeratedRange(Trap::Limit)) { + MOZ_TRY(CodePodVector(coder, &(*item)[trap])); + } + return Ok(); +} + +// WasmGC.h + +CoderResult CodeStackMap(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::StackMap, 12); + // Decode the stack map header + StackMapHeader header; + MOZ_TRY(CodePod(coder, &header)); + + // Allocate a stack map for the header + StackMap* map = StackMap::create(header); + if (!map) { + return Err(OutOfMemory()); + } + + // Decode the bitmap into the stackmap + MOZ_TRY(coder.readBytes(map->rawBitmap(), map->rawBitmapLengthInBytes())); + + *item = map; + return Ok(); +} + +template +CoderResult CodeStackMap(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::StackMap, 12); + STATIC_ASSERT_ENCODING_OR_SIZING; + + // Encode the stackmap header + MOZ_TRY(CodePod(coder, &item->header)); + + // Encode the stackmap bitmap + MOZ_TRY(coder.writeBytes(item->rawBitmap(), item->rawBitmapLengthInBytes())); + + return Ok(); +} + +static uint32_t ComputeCodeOffset(const uint8_t* codeStart, + const uint8_t* codePtr) { + MOZ_RELEASE_ASSERT(codePtr >= codeStart); +#ifdef JS_64BIT + MOZ_RELEASE_ASSERT(codePtr < codeStart + UINT32_MAX); +#endif + return (uint32_t)(codePtr - codeStart); +} + +CoderResult CodeStackMaps(Coder& coder, + CoderArg item, + const uint8_t* codeStart) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::StackMaps, 48); + // Decode the amount of stack maps + size_t length; + MOZ_TRY(CodePod(coder, &length)); + + for (size_t i = 0; i < length; i++) { + // Decode the offset relative to codeStart + uint32_t codeOffset; + MOZ_TRY(CodePod(coder, &codeOffset)); + + // Decode the stack map + StackMap* map; + MOZ_TRY(CodeStackMap(coder, &map)); + + // Add it to the map + const uint8_t* nextInsnAddr = codeStart + codeOffset; + if (!item->add(nextInsnAddr, map)) { + return Err(OutOfMemory()); + } + } + + // Finish the maps, asserting they are sorted + item->finishAlreadySorted(); + return Ok(); +} + +template +CoderResult CodeStackMaps(Coder& coder, + CoderArg item, + const uint8_t* codeStart) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::StackMaps, 48); + STATIC_ASSERT_ENCODING_OR_SIZING; + + // Encode the amount of stack maps + size_t length = item->length(); + MOZ_TRY(CodePod(coder, &length)); + + for (size_t i = 0; i < length; i++) { + StackMaps::Maplet maplet = item->get(i); + uint32_t codeOffset = ComputeCodeOffset(codeStart, maplet.nextInsnAddr); + + // Encode the offset relative to codeStart + MOZ_TRY(CodePod(coder, &codeOffset)); + + // Encode the stack map + MOZ_TRY(CodeStackMap(coder, maplet.map)); + } + return Ok(); +} + +// WasmCode.h + +template +CoderResult CodeSymbolicLinkArray( + Coder& coder, + CoderArg item) { + for (SymbolicAddress address : + mozilla::MakeEnumeratedRange(SymbolicAddress::Limit)) { + MOZ_TRY(CodePodVector(coder, &(*item)[address])); + } + return Ok(); +} + +template +CoderResult CodeLinkData(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::LinkData, 7608); + if constexpr (mode == MODE_ENCODE) { + MOZ_ASSERT(item->tier == Tier::Serialized); + } + MOZ_TRY(CodePod(coder, &item->pod())); + MOZ_TRY(CodePodVector(coder, &item->internalLinks)); + MOZ_TRY(CodeSymbolicLinkArray(coder, &item->symbolicLinks)); + return Ok(); +} + +CoderResult CodeModuleSegment(Coder& coder, + wasm::UniqueModuleSegment* item, + const wasm::LinkData& linkData) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::ModuleSegment, 48); + // Assert we're decoding a ModuleSegment + MOZ_TRY(Magic(coder, Marker::ModuleSegment)); + + // Decode the code bytes length + size_t length; + MOZ_TRY(CodePod(coder, &length)); + + // Allocate the code bytes + UniqueCodeBytes bytes = AllocateCodeBytes(length); + if (!bytes) { + return Err(OutOfMemory()); + } + + // Decode the code bytes + MOZ_TRY(coder.readBytes(bytes.get(), length)); + + // Initialize the ModuleSegment + *item = js::MakeUnique(Tier::Serialized, std::move(bytes), + length, linkData); + if (!*item) { + return Err(OutOfMemory()); + } + return Ok(); +} + +template +CoderResult CodeModuleSegment(Coder& coder, + CoderArg item, + const wasm::LinkData& linkData) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::ModuleSegment, 48); + STATIC_ASSERT_ENCODING_OR_SIZING; + MOZ_ASSERT((*item)->tier() == Tier::Serialized); + + // Mark that we're encoding a ModuleSegment + MOZ_TRY(Magic(coder, Marker::ModuleSegment)); + + // Encode the length + size_t length = (*item)->length(); + MOZ_TRY(CodePod(coder, &length)); + + if constexpr (mode == MODE_SIZE) { + // Just calculate the length of bytes written + MOZ_TRY(coder.writeBytes((*item)->base(), length)); + } else { + // Get the start of where the code bytes will be written + uint8_t* serializedBase = coder.buffer_; + + // Write the code bytes + MOZ_TRY(coder.writeBytes((*item)->base(), length)); + + // Unlink the code bytes written to the buffer + StaticallyUnlink(serializedBase, linkData); + } + return Ok(); +} + +template +CoderResult CodeMetadataTier(Coder& coder, + CoderArg item, + const uint8_t* codeStart) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::MetadataTier, 856); + MOZ_TRY(Magic(coder, Marker::MetadataTier)); + MOZ_TRY(CodePodVector(coder, &item->funcToCodeRange)); + MOZ_TRY(CodePodVector(coder, &item->codeRanges)); + MOZ_TRY(CodePodVector(coder, &item->callSites)); + MOZ_TRY(CodeTrapSiteVectorArray(coder, &item->trapSites)); + MOZ_TRY(CodePodVector(coder, &item->funcImports)); + MOZ_TRY(CodePodVector(coder, &item->funcExports)); + MOZ_TRY(CodeStackMaps(coder, &item->stackMaps, codeStart)); + MOZ_TRY(CodePodVector(coder, &item->tryNotes)); + return Ok(); +} + +template +CoderResult CodeMetadata(Coder& coder, + CoderArg item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Metadata, 408); + if constexpr (mode == MODE_ENCODE) { + // Serialization doesn't handle asm.js or debug enabled modules + MOZ_ASSERT(!item->debugEnabled && item->debugFuncTypeIndices.empty()); + MOZ_ASSERT(!item->isAsmJS()); + } + + MOZ_TRY(Magic(coder, Marker::Metadata)); + MOZ_TRY(CodePod(coder, &item->pod())); + MOZ_TRY((CodeRefPtr( + coder, &item->types))); + MOZ_TRY((CodeVector>( + coder, &item->globals))); + MOZ_TRY(( + CodeVector>(coder, &item->tables))); + MOZ_TRY((CodeVector>(coder, &item->tags))); + MOZ_TRY(CodePod(coder, &item->moduleName)); + MOZ_TRY(CodePodVector(coder, &item->funcNames)); + MOZ_TRY(CodeCacheableChars(coder, &item->filename)); + MOZ_TRY(CodeCacheableChars(coder, &item->sourceMapURL)); + + if constexpr (mode == MODE_DECODE) { + // Initialize debugging state to disabled + item->debugEnabled = false; + item->debugFuncTypeIndices.clear(); + } + + return Ok(); +} + +CoderResult CodeCodeTier(Coder& coder, wasm::UniqueCodeTier* item, + const wasm::LinkData& linkData) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::CodeTier, 248); + UniqueMetadataTier metadata; + UniqueModuleSegment segment; + MOZ_TRY(Magic(coder, Marker::CodeTier)); + MOZ_TRY(CodeModuleSegment(coder, &segment, linkData)); + MOZ_TRY((CodeUniquePtr( + coder, &metadata, &CodeMetadataTier, segment->base()))); + *item = js::MakeUnique(std::move(metadata), std::move(segment)); + if (!*item) { + return Err(OutOfMemory()); + } + return Ok(); +} + +template +CoderResult CodeCodeTier(Coder& coder, + CoderArg item, + const wasm::LinkData& linkData) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::CodeTier, 248); + STATIC_ASSERT_ENCODING_OR_SIZING; + MOZ_TRY(Magic(coder, Marker::CodeTier)); + MOZ_TRY(CodeModuleSegment(coder, &item->segment_, linkData)); + MOZ_TRY((CodeUniquePtr(coder, &item->metadata_, + &CodeMetadataTier, + item->segment_->base()))); + return Ok(); +} + +CoderResult CodeSharedCode(Coder& coder, wasm::SharedCode* item, + const wasm::LinkData& linkData, + const wasm::CustomSectionVector& customSections) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Code, 192); + MutableMetadata metadata; + UniqueCodeTier codeTier; + MOZ_TRY((CodeRefPtr(coder, &metadata))); + MOZ_TRY(CodeCodeTier(coder, &codeTier, linkData)); + + // Initialize metadata's name payload from the custom section + if (metadata->nameCustomSectionIndex) { + metadata->namePayload = + customSections[*metadata->nameCustomSectionIndex].payload; + } else { + MOZ_RELEASE_ASSERT(!metadata->moduleName); + MOZ_RELEASE_ASSERT(metadata->funcNames.empty()); + } + + // Initialize the jump tables + JumpTables jumpTables; + if (!jumpTables.init(CompileMode::Once, codeTier->segment(), + codeTier->metadata().codeRanges)) { + return Err(OutOfMemory()); + } + + // Create and initialize the code + MutableCode code = + js_new(std::move(codeTier), *metadata, std::move(jumpTables)); + if (!code || !code->initialize(linkData)) { + return Err(OutOfMemory()); + } + *item = code; + return Ok(); +} + +template +CoderResult CodeSharedCode(Coder& coder, + CoderArg item, + const wasm::LinkData& linkData) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Code, 192); + STATIC_ASSERT_ENCODING_OR_SIZING; + MOZ_TRY((CodeRefPtr( + coder, &(*item)->metadata_))); + MOZ_TRY(CodeCodeTier(coder, &(*item)->codeTier(Tier::Serialized), linkData)); + return Ok(); +} + +// WasmModule.h + +CoderResult CodeModule(Coder& coder, MutableModule* item) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Module, 256); + JS::BuildIdCharVector currentBuildId; + if (!GetOptimizedEncodingBuildId(¤tBuildId)) { + return Err(OutOfMemory()); + } + JS::BuildIdCharVector deserializedBuildId; + MOZ_TRY(CodePodVector(coder, &deserializedBuildId)); + + MOZ_RELEASE_ASSERT(EqualContainers(currentBuildId, deserializedBuildId)); + + LinkData linkData(Tier::Serialized); + MOZ_TRY(Magic(coder, Marker::LinkData)); + MOZ_TRY(CodeLinkData(coder, &linkData)); + + ImportVector imports; + MOZ_TRY(Magic(coder, Marker::Imports)); + MOZ_TRY((CodeVector>( + coder, &imports))); + + ExportVector exports; + MOZ_TRY(Magic(coder, Marker::Exports)); + MOZ_TRY((CodeVector>( + coder, &exports))); + + DataSegmentVector dataSegments; + MOZ_TRY(Magic(coder, Marker::DataSegments)); + MOZ_TRY((CodeVector>>( + coder, &dataSegments))); + + ElemSegmentVector elemSegments; + MOZ_TRY(Magic(coder, Marker::ElemSegments)); + MOZ_TRY((CodeVector>>( + coder, &elemSegments))); + + CustomSectionVector customSections; + MOZ_TRY(Magic(coder, Marker::CustomSections)); + MOZ_TRY( + (CodeVector>( + coder, &customSections))); + + SharedCode code; + MOZ_TRY(Magic(coder, Marker::Code)); + MOZ_TRY(CodeSharedCode(coder, &code, linkData, customSections)); + + *item = js_new(*code, std::move(imports), std::move(exports), + std::move(dataSegments), std::move(elemSegments), + std::move(customSections), nullptr, + /* loggingDeserialized = */ true); + return Ok(); +} + +template +CoderResult CodeModule(Coder& coder, CoderArg item, + const wasm::LinkData& linkData) { + WASM_VERIFY_SERIALIZATION_FOR_SIZE(wasm::Module, 256); + STATIC_ASSERT_ENCODING_OR_SIZING; + MOZ_RELEASE_ASSERT(!item->metadata().debugEnabled); + MOZ_RELEASE_ASSERT(item->code_->hasTier(Tier::Serialized)); + + JS::BuildIdCharVector currentBuildId; + if (!GetOptimizedEncodingBuildId(¤tBuildId)) { + return Err(OutOfMemory()); + } + MOZ_TRY(CodePodVector(coder, ¤tBuildId)); + MOZ_TRY(Magic(coder, Marker::LinkData)); + MOZ_TRY(CodeLinkData(coder, &linkData)); + MOZ_TRY(Magic(coder, Marker::Imports)); + MOZ_TRY( + (CodeVector>(coder, &item->imports_))); + MOZ_TRY(Magic(coder, Marker::Exports)); + MOZ_TRY( + (CodeVector>(coder, &item->exports_))); + MOZ_TRY(Magic(coder, Marker::DataSegments)); + MOZ_TRY( + (CodeVector>>( + coder, &item->dataSegments_))); + MOZ_TRY(Magic(coder, Marker::ElemSegments)); + MOZ_TRY( + (CodeVector>>( + coder, &item->elemSegments_))); + MOZ_TRY(Magic(coder, Marker::CustomSections)); + MOZ_TRY((CodeVector>( + coder, &item->customSections_))); + MOZ_TRY(Magic(coder, Marker::Code)); + MOZ_TRY(CodeSharedCode(coder, &item->code_, linkData)); + return Ok(); +} + +} // namespace wasm +} // namespace js + +static bool GetSerializedSize(const Module& module, const LinkData& linkData, + size_t* size) { + Coder coder(module.metadata().types.get()); + auto result = CodeModule(coder, &module, linkData); + if (result.isErr()) { + return false; + } + *size = coder.size_.value(); + return true; +} + +bool Module::serialize(const LinkData& linkData, Bytes* bytes) const { + MOZ_RELEASE_ASSERT(!metadata().debugEnabled); + MOZ_RELEASE_ASSERT(code_->hasTier(Tier::Serialized)); + + size_t serializedSize; + if (!GetSerializedSize(*this, linkData, &serializedSize)) { + // An error is an overflow, return false + return false; + } + + // Try to allocate the destination buffer + if (!bytes->resizeUninitialized(serializedSize)) { + return false; + } + + Coder coder(metadata().types.get(), bytes->begin(), + serializedSize); + CoderResult result = CodeModule(coder, this, linkData); + if (result.isErr()) { + // An error is an OOM, return false + return false; + } + // Every byte is accounted for + MOZ_RELEASE_ASSERT(coder.buffer_ == coder.end_); + return true; +} + +/* static */ +MutableModule Module::deserialize(const uint8_t* begin, size_t size) { + Coder coder(begin, size); + MutableModule module; + CoderResult result = CodeModule(coder, &module); + if (result.isErr()) { + // An error is an OOM, return nullptr + return nullptr; + } + // Every byte is accounted for + MOZ_RELEASE_ASSERT(coder.buffer_ == coder.end_); + return module; +} + +void Module::initGCMallocBytesExcludingCode() { + // The size doesn't have to be exact so use the serialization framework to + // calculate a value. We consume all errors, as they can only be overflow and + // can be ignored until the end. + constexpr CoderMode MODE = MODE_SIZE; + Coder coder(metadata().types.get()); + (void)CodeVector>(coder, &imports_); + (void)CodeVector>(coder, &exports_); + (void)CodeVector>>( + coder, &dataSegments_); + (void)CodeVector>>( + coder, &elemSegments_); + (void)CodeVector>( + coder, &customSections_); + + // Overflow really shouldn't be possible here, but handle it anyways. + size_t serializedSize = coder.size_.isValid() ? coder.size_.value() : 0; + gcMallocBytesExcludingCode_ = sizeof(*this) + serializedSize; +} diff --git a/js/src/wasm/WasmSerialize.h b/js/src/wasm/WasmSerialize.h new file mode 100644 index 0000000000..918e50b42d --- /dev/null +++ b/js/src/wasm/WasmSerialize.h @@ -0,0 +1,296 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2022 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_serialize_h +#define wasm_serialize_h + +#include "mozilla/CheckedInt.h" +#include "mozilla/MacroForEach.h" +#include "mozilla/Maybe.h" +#include "mozilla/Result.h" + +#include +#include +#include + +namespace js { +namespace wasm { + +class TypeContext; + +// [SMDOC] "Module serialization" +// +// A wasm::Module may be serialized to a binary format that allows for quick +// reloads of a previous compiled wasm binary. +// +// The binary format is optimized for encoding/decoding speed, not size. There +// is no formal specification, and no backwards/forwards compatibility +// guarantees. The prelude of the encoding contains a 'build ID' which must be +// used when reading from a cache entry to determine if it is valid. +// +// Module serialization and deserialization are performed using templated +// functions that allow for (imperfect) abstraction over whether we are decoding +// or encoding the module. It can be viewed as a specialization of the visitor +// pattern. +// +// Each module data structure is visited by a function parameterized by the +// "mode", which may be either: +// 1. MODE_SIZE - We are computing the final encoding size, before encoding it +// 2. MODE_ENCODE - We are actually encoding the module to bytes +// 3. MODE_DECODE - We are decoding the module from bytes +// +// These functions are called "coding" functions, as they are generic to whether +// we are "encoding" or "decoding". The verb tense "code" is used for the +// prefix. +// +// Each coding function takes the item being visited, along with a "Coder" +// which contains the state needed for each mode. This is either a buffer span +// or an accumulated length. The coding function either manipulates the Coder +// directly or delegates to its field's coding functions. +// +// Leaf data types are usually just copied directly to and from memory using a +// generic "CodePod" function. See the "cacheable POD" documentation in this +// file for more information. +// +// Non-leaf data types need an explicit coding function. This function can +// usually be completely generic to decoding/encoding, and delegate to the +// coding functions for each field. Separate decoding/encoding functions may +// be needed when decoding requires initialization logic, such as constructors. +// In this case, it is critical that both functions agree on the fields to be +// coded, and the order they are coded in. +// +// Coding functions are defined as free functions in "WasmSerialize.cpp". When +// they require access to protected state in a type, they may use the +// WASM_DECLARE_FRIEND_SERIALIZE macro. + +// Signal an out of memory condition +struct OutOfMemory {}; + +// The result of serialization, either OK or OOM +using CoderResult = mozilla::Result; + +// CoderMode parameterizes the coding functions +enum CoderMode { + // We are computing the final size of the encoded buffer. This is a discrete + // pass that runs before encoding. + MODE_SIZE, + // We are encoding the module to bytes. + MODE_ENCODE, + // We are decoding the module from bytes. + MODE_DECODE, +}; + +// Coding functions take a different argument depending on which CoderMode +// they are invoked with: +// * MODE_SIZE - const T* +// * MODE_ENCODE - const T* +// * MODE_DECODE - T* +// +// The CoderArg type alias is used to acquire the proper type for +// coding function arguments. +template +struct CoderArgT; + +template +struct CoderArgT { + using T = const V*; +}; + +template +struct CoderArgT { + using T = V*; +}; + +template +struct CoderArgT { + using T = const V*; +}; + +template +using CoderArg = typename CoderArgT::T; + +// Coder is the state provided to all coding functions during module traversal. +template +struct Coder; + +// A Coder computes the total encoded size of a module +template <> +struct Coder { + explicit Coder(const TypeContext* types) : types_(types), size_(0) {} + + // The types of the module that we're going to encode. This is required in + // order to encode the original index of types that we encounter. + const TypeContext* types_; + + // The current size of buffer required to serialize this module. + mozilla::CheckedInt size_; + + // This function shares a signature with MODE_ENCODE to allow functions to be + // generic across MODE_SIZE/MODE_ENCODE, even though the src pointer is not + // needed for MODE_SIZE. + CoderResult writeBytes(const void* unusedSrc, size_t length); +}; + +// A Coder holds the buffer being written to +template <> +struct Coder { + Coder(const TypeContext* types, uint8_t* start, size_t length) + : types_(types), buffer_(start), end_(start + length) {} + + // The types of the module that we're encoding. This is required in + // order to encode the original index of types that we encounter. + const TypeContext* types_; + + // The current position in the buffer we're writing to. + uint8_t* buffer_; + // The end position in the buffer we're writing to. + const uint8_t* end_; + + CoderResult writeBytes(const void* src, size_t length); +}; + +// A Coder holds the buffer being read from +template <> +struct Coder { + Coder(const uint8_t* start, size_t length) + : types_(nullptr), buffer_(start), end_(start + length) {} + + // The types of the module that we're decoding. This is null until the types + // of this module are decoded. + const TypeContext* types_; + + // The current position in the buffer we're reading from. + const uint8_t* buffer_; + // The end position in the buffer we're reading from. + const uint8_t* end_; + + CoderResult readBytes(void* dest, size_t length); +}; + +// Macros to help types declare friendship with a coding function + +#define WASM_DECLARE_FRIEND_SERIALIZE(TYPE) \ + template \ + friend CoderResult Code##TYPE(Coder&, CoderArg); + +#define WASM_DECLARE_FRIEND_SERIALIZE_ARGS(TYPE, ARGS...) \ + template \ + friend CoderResult Code##TYPE(Coder&, CoderArg, ARGS); + +// [SMDOC] "Cacheable POD" +// +// Module serialization relies on copying simple structs to and from the +// cache format. We need a way to ensure that we only do this on types that are +// "safe". We call this "cacheable POD". Note: this is not the same thing as +// "POD" as that may contain pointers, which are not cacheable. +// +// We define cacheable POD (C-POD) recursively upon types: +// 1. any integer type is C-POD +// 2. any floating point type is C-POD +// 3. any enum type is C-POD +// 4. any mozilla::Maybe with T: C-POD is C-POD +// 5. any T[N] with T: C-POD is C-POD +// 6. any union where all fields are C-POD is C-POD +// 7. any struct with the following conditions must is C-POD +// * every field's type must be C-POD +// * the parent type, if it exists, must also be C-POD +// * there must be no virtual methods +// +// There are no combination of C++ type traits at this time that can +// automatically meet these criteria, so we are rolling our own system. +// +// We define a "IsCacheablePod" type trait, with builtin rules for cases (1-5). +// The complex cases (6-7) are handled using manual declaration and checking +// macros that must be used upon structs and unions that are considered +// cacheable POD. +// +// See the following macros for details: +// - WASM_DECLARE_CACHEABLE_POD +// - WASM_CHECK_CACHEABLE_POD[_WITH_PARENT] + +// The IsCacheablePod type trait primary template. Contains the rules for +// (cases 1-3). +template +struct IsCacheablePod + : public std::conditional_t || std::is_enum_v, + std::true_type, std::false_type> {}; + +// Partial specialization for (case 4). +template +struct IsCacheablePod> + : public std::conditional_t::value, std::true_type, + std::false_type> {}; + +// Partial specialization for (case 5). +template +struct IsCacheablePod + : public std::conditional_t::value, std::true_type, + std::false_type> {}; + +template +inline constexpr bool is_cacheable_pod = IsCacheablePod::value; + +// Checks if derrived class will not use the structure alignment for its +// next field. It used when pod is a base class. +#define WASM_CHECK_CACHEABLE_POD_PADDING(Type) \ + class __CHECK_PADING_##Type : public Type { \ + public: \ + char c; \ + }; \ + static_assert(sizeof(__CHECK_PADING_##Type) > sizeof(Type), \ + #Type " will overlap with next field if inherited"); + +// Declare the type 'Type' to be cacheable POD. The definition of the type must +// contain a WASM_CHECK_CACHEABLE_POD[_WITH_PARENT] to ensure all fields of the +// type are cacheable POD. +#define WASM_DECLARE_CACHEABLE_POD(Type) \ + static_assert(!std::is_polymorphic_v, \ + #Type "must not have virtual methods"); \ + } /* namespace wasm */ \ + } /* namespace js */ \ + template <> \ + struct js::wasm::IsCacheablePod : public std::true_type {}; \ + namespace js { \ + namespace wasm { + +// Helper: check each field's type to be cacheable POD +#define WASM_CHECK_CACHEABLE_POD_FIELD_(Field) \ + static_assert(js::wasm::IsCacheablePod::value, \ + #Field " must be cacheable pod"); + +// Check every field in a type definition to ensure they are cacheable POD. +#define WASM_CHECK_CACHEABLE_POD(Fields...) \ + MOZ_FOR_EACH(WASM_CHECK_CACHEABLE_POD_FIELD_, (), (Fields)) + +// Check every field in a type definition to ensure they are cacheable POD, and +// check that the parent class is also cacheable POD. +#define WASM_CHECK_CACHEABLE_POD_WITH_PARENT(Parent, Fields...) \ + static_assert(js::wasm::IsCacheablePod::value, \ + #Parent " must be cacheable pod"); \ + MOZ_FOR_EACH(WASM_CHECK_CACHEABLE_POD_FIELD_, (), (Fields)) + +// Allow fields that are not cacheable POD but are believed to be safe for +// serialization due to some justification. +#define WASM_ALLOW_NON_CACHEABLE_POD_FIELD(Field, Reason) \ + static_assert(!js::wasm::IsCacheablePod::value, \ + #Field " is not cacheable due to " Reason); + +} // namespace wasm +} // namespace js + +#endif // wasm_serialize_h diff --git a/js/src/wasm/WasmShareable.h b/js/src/wasm/WasmShareable.h new file mode 100644 index 0000000000..a2df641c3f --- /dev/null +++ b/js/src/wasm/WasmShareable.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_shareable_h +#define wasm_shareable_h + +#include "mozilla/RefPtr.h" +#include "js/RefCounted.h" +#include "wasm/WasmTypeDecls.h" + +namespace js { +namespace wasm { + +using mozilla::MallocSizeOf; + +// This reusable base class factors out the logic for a resource that is shared +// by multiple instances/modules but should only be counted once when computing +// about:memory stats. + +template +using SeenSet = HashSet, SystemAllocPolicy>; + +template +struct ShareableBase : AtomicRefCounted { + using SeenSet = wasm::SeenSet; + + size_t sizeOfIncludingThisIfNotSeen(MallocSizeOf mallocSizeOf, + SeenSet* seen) const { + const T* self = static_cast(this); + typename SeenSet::AddPtr p = seen->lookupForAdd(self); + if (p) { + return 0; + } + bool ok = seen->add(p, self); + (void)ok; // oh well + return mallocSizeOf(self) + self->sizeOfExcludingThis(mallocSizeOf); + } +}; + +// ShareableBytes is a reference-counted Vector of bytes. + +struct ShareableBytes : ShareableBase { + // Vector is 'final', so instead make Vector a member and add boilerplate. + Bytes bytes; + + ShareableBytes() = default; + explicit ShareableBytes(Bytes&& bytes) : bytes(std::move(bytes)) {} + size_t sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return bytes.sizeOfExcludingThis(mallocSizeOf); + } + const uint8_t* begin() const { return bytes.begin(); } + const uint8_t* end() const { return bytes.end(); } + size_t length() const { return bytes.length(); } + bool append(const uint8_t* start, size_t len) { + return bytes.append(start, len); + } +}; + +using MutableBytes = RefPtr; +using SharedBytes = RefPtr; + +} // namespace wasm +} // namespace js + +#endif // wasm_shareable_h diff --git a/js/src/wasm/WasmSignalHandlers.cpp b/js/src/wasm/WasmSignalHandlers.cpp new file mode 100644 index 0000000000..86e11df7f0 --- /dev/null +++ b/js/src/wasm/WasmSignalHandlers.cpp @@ -0,0 +1,1084 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2014 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmSignalHandlers.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/ThreadLocal.h" + +#include "threading/Thread.h" +#include "vm/JitActivation.h" // js::jit::JitActivation +#include "vm/Realm.h" +#include "vm/Runtime.h" +#include "wasm/WasmCode.h" +#include "wasm/WasmInstance.h" + +#if defined(XP_WIN) +# include // must include before util/WindowsWrapper.h's `#undef`s +# include "util/WindowsWrapper.h" +#elif defined(XP_DARWIN) +# include +# include +#else +# include +#endif + +using namespace js; +using namespace js::wasm; + +using mozilla::DebugOnly; + +#if !defined(JS_CODEGEN_NONE) + +// ============================================================================= +// This following pile of macros and includes defines the ToRegisterState() and +// the ContextTo{PC,FP,SP,LR}() functions from the (highly) platform-specific +// CONTEXT struct which is provided to the signal handler. +// ============================================================================= + +# if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) +# include // for ucontext_t, mcontext_t +# endif + +# if defined(__x86_64__) +# if defined(__DragonFly__) +# include // for union savefpu +# elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || \ + defined(__NetBSD__) || defined(__OpenBSD__) +# include // for struct savefpu/fxsave64 +# endif +# endif + +# if defined(XP_WIN) +# define EIP_sig(p) ((p)->Eip) +# define EBP_sig(p) ((p)->Ebp) +# define ESP_sig(p) ((p)->Esp) +# define RIP_sig(p) ((p)->Rip) +# define RSP_sig(p) ((p)->Rsp) +# define RBP_sig(p) ((p)->Rbp) +# define R11_sig(p) ((p)->R11) +# define R13_sig(p) ((p)->R13) +# define R14_sig(p) ((p)->R14) +# define R15_sig(p) ((p)->R15) +# define EPC_sig(p) ((p)->Pc) +# define RFP_sig(p) ((p)->Fp) +# define R31_sig(p) ((p)->Sp) +# define RLR_sig(p) ((p)->Lr) +# elif defined(__OpenBSD__) +# define EIP_sig(p) ((p)->sc_eip) +# define EBP_sig(p) ((p)->sc_ebp) +# define ESP_sig(p) ((p)->sc_esp) +# define RIP_sig(p) ((p)->sc_rip) +# define RSP_sig(p) ((p)->sc_rsp) +# define RBP_sig(p) ((p)->sc_rbp) +# define R11_sig(p) ((p)->sc_r11) +# if defined(__arm__) +# define R13_sig(p) ((p)->sc_usr_sp) +# define R14_sig(p) ((p)->sc_usr_lr) +# define R15_sig(p) ((p)->sc_pc) +# else +# define R13_sig(p) ((p)->sc_r13) +# define R14_sig(p) ((p)->sc_r14) +# define R15_sig(p) ((p)->sc_r15) +# endif +# if defined(__aarch64__) +# define EPC_sig(p) ((p)->sc_elr) +# define RFP_sig(p) ((p)->sc_x[29]) +# define RLR_sig(p) ((p)->sc_lr) +# define R31_sig(p) ((p)->sc_sp) +# endif +# if defined(__mips__) +# define EPC_sig(p) ((p)->sc_pc) +# define RFP_sig(p) ((p)->sc_regs[30]) +# endif +# if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ + defined(__PPC64LE__) +# define R01_sig(p) ((p)->sc_frame.fixreg[1]) +# define R32_sig(p) ((p)->sc_frame.srr0) +# endif +# elif defined(__linux__) || defined(__sun) +# if defined(__linux__) +# define EIP_sig(p) ((p)->uc_mcontext.gregs[REG_EIP]) +# define EBP_sig(p) ((p)->uc_mcontext.gregs[REG_EBP]) +# define ESP_sig(p) ((p)->uc_mcontext.gregs[REG_ESP]) +# else +# define EIP_sig(p) ((p)->uc_mcontext.gregs[REG_PC]) +# define EBP_sig(p) ((p)->uc_mcontext.gregs[REG_EBP]) +# define ESP_sig(p) ((p)->uc_mcontext.gregs[REG_ESP]) +# endif +# define RIP_sig(p) ((p)->uc_mcontext.gregs[REG_RIP]) +# define RSP_sig(p) ((p)->uc_mcontext.gregs[REG_RSP]) +# define RBP_sig(p) ((p)->uc_mcontext.gregs[REG_RBP]) +# if defined(__linux__) && defined(__arm__) +# define R11_sig(p) ((p)->uc_mcontext.arm_fp) +# define R13_sig(p) ((p)->uc_mcontext.arm_sp) +# define R14_sig(p) ((p)->uc_mcontext.arm_lr) +# define R15_sig(p) ((p)->uc_mcontext.arm_pc) +# else +# define R11_sig(p) ((p)->uc_mcontext.gregs[REG_R11]) +# define R13_sig(p) ((p)->uc_mcontext.gregs[REG_R13]) +# define R14_sig(p) ((p)->uc_mcontext.gregs[REG_R14]) +# define R15_sig(p) ((p)->uc_mcontext.gregs[REG_R15]) +# endif +# if defined(__linux__) && defined(__aarch64__) +# define EPC_sig(p) ((p)->uc_mcontext.pc) +# define RFP_sig(p) ((p)->uc_mcontext.regs[29]) +# define RLR_sig(p) ((p)->uc_mcontext.regs[30]) +# define R31_sig(p) ((p)->uc_mcontext.sp) +# endif +# if defined(__linux__) && defined(__mips__) +# define EPC_sig(p) ((p)->uc_mcontext.pc) +# define RFP_sig(p) ((p)->uc_mcontext.gregs[30]) +# define RSP_sig(p) ((p)->uc_mcontext.gregs[29]) +# define R31_sig(p) ((p)->uc_mcontext.gregs[31]) +# endif +# if defined(__linux__) && (defined(__sparc__) && defined(__arch64__)) +# define PC_sig(p) ((p)->uc_mcontext.mc_gregs[MC_PC]) +# define FP_sig(p) ((p)->uc_mcontext.mc_fp) +# define SP_sig(p) ((p)->uc_mcontext.mc_i7) +# endif +# if defined(__linux__) && (defined(__ppc64__) || defined(__PPC64__) || \ + defined(__ppc64le__) || defined(__PPC64LE__)) +# define R01_sig(p) ((p)->uc_mcontext.gp_regs[1]) +# define R32_sig(p) ((p)->uc_mcontext.gp_regs[32]) +# endif +# if defined(__linux__) && defined(__loongarch__) +# define EPC_sig(p) ((p)->uc_mcontext.__pc) +# define RRA_sig(p) ((p)->uc_mcontext.__gregs[1]) +# define R03_sig(p) ((p)->uc_mcontext.__gregs[3]) +# define RFP_sig(p) ((p)->uc_mcontext.__gregs[22]) +# endif +# if defined(__linux__) && defined(__riscv) +# define RPC_sig(p) ((p)->uc_mcontext.__gregs[REG_PC]) +# define RRA_sig(p) ((p)->uc_mcontext.__gregs[REG_RA]) +# define RFP_sig(p) ((p)->uc_mcontext.__gregs[REG_S0]) +# define R02_sig(p) ((p)->uc_mcontext.__gregs[REG_SP]) +# endif +# if defined(__sun__) && defined(__sparc__) +# define PC_sig(p) ((p)->uc_mcontext.gregs[REG_PC]) +# define FP_sig(p) ((p)->uc_mcontext.gregs[REG_FPRS]) +# define SP_sig(p) ((p)->uc_mcontext.gregs[REG_SP]) +# endif +# elif defined(__NetBSD__) +# define EIP_sig(p) ((p)->uc_mcontext.__gregs[_REG_EIP]) +# define EBP_sig(p) ((p)->uc_mcontext.__gregs[_REG_EBP]) +# define ESP_sig(p) ((p)->uc_mcontext.__gregs[_REG_ESP]) +# define RIP_sig(p) ((p)->uc_mcontext.__gregs[_REG_RIP]) +# define RSP_sig(p) ((p)->uc_mcontext.__gregs[_REG_RSP]) +# define RBP_sig(p) ((p)->uc_mcontext.__gregs[_REG_RBP]) +# define R11_sig(p) ((p)->uc_mcontext.__gregs[_REG_R11]) +# define R13_sig(p) ((p)->uc_mcontext.__gregs[_REG_R13]) +# define R14_sig(p) ((p)->uc_mcontext.__gregs[_REG_R14]) +# define R15_sig(p) ((p)->uc_mcontext.__gregs[_REG_R15]) +# if defined(__aarch64__) +# define EPC_sig(p) ((p)->uc_mcontext.__gregs[_REG_PC]) +# define RFP_sig(p) ((p)->uc_mcontext.__gregs[_REG_X29]) +# define RLR_sig(p) ((p)->uc_mcontext.__gregs[_REG_X30]) +# define R31_sig(p) ((p)->uc_mcontext.__gregs[_REG_SP]) +# endif +# if defined(__mips__) +# define EPC_sig(p) ((p)->uc_mcontext.__gregs[_REG_EPC]) +# define RFP_sig(p) ((p)->uc_mcontext.__gregs[_REG_S8]) +# endif +# if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ + defined(__PPC64LE__) +# define R01_sig(p) ((p)->uc_mcontext.__gregs[_REG_R1]) +# define R32_sig(p) ((p)->uc_mcontext.__gregs[_REG_PC]) +# endif +# elif defined(__DragonFly__) || defined(__FreeBSD__) || \ + defined(__FreeBSD_kernel__) +# define EIP_sig(p) ((p)->uc_mcontext.mc_eip) +# define EBP_sig(p) ((p)->uc_mcontext.mc_ebp) +# define ESP_sig(p) ((p)->uc_mcontext.mc_esp) +# define RIP_sig(p) ((p)->uc_mcontext.mc_rip) +# define RSP_sig(p) ((p)->uc_mcontext.mc_rsp) +# define RBP_sig(p) ((p)->uc_mcontext.mc_rbp) +# if defined(__FreeBSD__) && defined(__arm__) +# define R11_sig(p) ((p)->uc_mcontext.__gregs[_REG_R11]) +# define R13_sig(p) ((p)->uc_mcontext.__gregs[_REG_R13]) +# define R14_sig(p) ((p)->uc_mcontext.__gregs[_REG_R14]) +# define R15_sig(p) ((p)->uc_mcontext.__gregs[_REG_R15]) +# else +# define R11_sig(p) ((p)->uc_mcontext.mc_r11) +# define R13_sig(p) ((p)->uc_mcontext.mc_r13) +# define R14_sig(p) ((p)->uc_mcontext.mc_r14) +# define R15_sig(p) ((p)->uc_mcontext.mc_r15) +# endif +# if defined(__FreeBSD__) && defined(__aarch64__) +# define EPC_sig(p) ((p)->uc_mcontext.mc_gpregs.gp_elr) +# define RFP_sig(p) ((p)->uc_mcontext.mc_gpregs.gp_x[29]) +# define RLR_sig(p) ((p)->uc_mcontext.mc_gpregs.gp_lr) +# define R31_sig(p) ((p)->uc_mcontext.mc_gpregs.gp_sp) +# endif +# if defined(__FreeBSD__) && defined(__mips__) +# define EPC_sig(p) ((p)->uc_mcontext.mc_pc) +# define RFP_sig(p) ((p)->uc_mcontext.mc_regs[30]) +# endif +# if defined(__FreeBSD__) && (defined(__ppc64__) || defined(__PPC64__) || \ + defined(__ppc64le__) || defined(__PPC64LE__)) +# define R01_sig(p) ((p)->uc_mcontext.mc_gpr[1]) +# define R32_sig(p) ((p)->uc_mcontext.mc_srr0) +# endif +# elif defined(XP_DARWIN) +# define EIP_sig(p) ((p)->thread.uts.ts32.__eip) +# define EBP_sig(p) ((p)->thread.uts.ts32.__ebp) +# define ESP_sig(p) ((p)->thread.uts.ts32.__esp) +# define RIP_sig(p) ((p)->thread.__rip) +# define RBP_sig(p) ((p)->thread.__rbp) +# define RSP_sig(p) ((p)->thread.__rsp) +# define R11_sig(p) ((p)->thread.__r[11]) +# define R13_sig(p) ((p)->thread.__sp) +# define R14_sig(p) ((p)->thread.__lr) +# define R15_sig(p) ((p)->thread.__pc) +# define EPC_sig(p) ((p)->thread.__pc) +# define RFP_sig(p) ((p)->thread.__fp) +# define R31_sig(p) ((p)->thread.__sp) +# define RLR_sig(p) ((p)->thread.__lr) +# else +# error \ + "Don't know how to read/write to the thread state via the mcontext_t." +# endif + +# if defined(ANDROID) +// Not all versions of the Android NDK define ucontext_t or mcontext_t. +// Detect this and provide custom but compatible definitions. Note that these +// follow the GLibc naming convention to access register values from +// mcontext_t. +// +// See: https://chromiumcodereview.appspot.com/10829122/ +// See: http://code.google.com/p/android/issues/detail?id=34784 +# if !defined(__BIONIC_HAVE_UCONTEXT_T) +# if defined(__arm__) + +// GLibc on ARM defines mcontext_t has a typedef for 'struct sigcontext'. +// Old versions of the C library didn't define the type. +# if !defined(__BIONIC_HAVE_STRUCT_SIGCONTEXT) +# include +# endif + +typedef struct sigcontext mcontext_t; + +typedef struct ucontext { + uint32_t uc_flags; + struct ucontext* uc_link; + stack_t uc_stack; + mcontext_t uc_mcontext; + // Other fields are not used so don't define them here. +} ucontext_t; + +# elif defined(__mips__) + +typedef struct { + uint32_t regmask; + uint32_t status; + uint64_t pc; + uint64_t gregs[32]; + uint64_t fpregs[32]; + uint32_t acx; + uint32_t fpc_csr; + uint32_t fpc_eir; + uint32_t used_math; + uint32_t dsp; + uint64_t mdhi; + uint64_t mdlo; + uint32_t hi1; + uint32_t lo1; + uint32_t hi2; + uint32_t lo2; + uint32_t hi3; + uint32_t lo3; +} mcontext_t; + +typedef struct ucontext { + uint32_t uc_flags; + struct ucontext* uc_link; + stack_t uc_stack; + mcontext_t uc_mcontext; + // Other fields are not used so don't define them here. +} ucontext_t; + +# elif defined(__loongarch64) + +typedef struct { + uint64_t pc; + uint64_t gregs[32]; + uint64_t fpregs[32]; + uint32_t fpc_csr; +} mcontext_t; + +typedef struct ucontext { + uint32_t uc_flags; + struct ucontext* uc_link; + stack_t uc_stack; + mcontext_t uc_mcontext; + // Other fields are not used so don't define them here. +} ucontext_t; + +# elif defined(__i386__) +// x86 version for Android. +typedef struct { + uint32_t gregs[19]; + void* fpregs; + uint32_t oldmask; + uint32_t cr2; +} mcontext_t; + +typedef uint32_t kernel_sigset_t[2]; // x86 kernel uses 64-bit signal masks +typedef struct ucontext { + uint32_t uc_flags; + struct ucontext* uc_link; + stack_t uc_stack; + mcontext_t uc_mcontext; + // Other fields are not used by V8, don't define them here. +} ucontext_t; +enum { REG_EIP = 14 }; +# endif // defined(__i386__) +# endif // !defined(__BIONIC_HAVE_UCONTEXT_T) +# endif // defined(ANDROID) + +# if defined(XP_DARWIN) +# if defined(__x86_64__) +struct macos_x64_context { + x86_thread_state64_t thread; + x86_float_state64_t float_; +}; +# define CONTEXT macos_x64_context +# elif defined(__i386__) +struct macos_x86_context { + x86_thread_state_t thread; + x86_float_state_t float_; +}; +# define CONTEXT macos_x86_context +# elif defined(__arm__) +struct macos_arm_context { + arm_thread_state_t thread; + arm_neon_state_t float_; +}; +# define CONTEXT macos_arm_context +# elif defined(__aarch64__) +struct macos_aarch64_context { + arm_thread_state64_t thread; + arm_neon_state64_t float_; +}; +# define CONTEXT macos_aarch64_context +# else +# error Unsupported architecture +# endif +# elif !defined(XP_WIN) +# define CONTEXT ucontext_t +# endif + +# if defined(_M_X64) || defined(__x86_64__) +# define PC_sig(p) RIP_sig(p) +# define FP_sig(p) RBP_sig(p) +# define SP_sig(p) RSP_sig(p) +# elif defined(_M_IX86) || defined(__i386__) +# define PC_sig(p) EIP_sig(p) +# define FP_sig(p) EBP_sig(p) +# define SP_sig(p) ESP_sig(p) +# elif defined(__arm__) +# define FP_sig(p) R11_sig(p) +# define SP_sig(p) R13_sig(p) +# define LR_sig(p) R14_sig(p) +# define PC_sig(p) R15_sig(p) +# elif defined(_M_ARM64) || defined(__aarch64__) +# define PC_sig(p) EPC_sig(p) +# define FP_sig(p) RFP_sig(p) +# define SP_sig(p) R31_sig(p) +# define LR_sig(p) RLR_sig(p) +# elif defined(__mips__) +# define PC_sig(p) EPC_sig(p) +# define FP_sig(p) RFP_sig(p) +# define SP_sig(p) RSP_sig(p) +# define LR_sig(p) R31_sig(p) +# elif defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ + defined(__PPC64LE__) +# define PC_sig(p) R32_sig(p) +# define SP_sig(p) R01_sig(p) +# define FP_sig(p) R01_sig(p) +# elif defined(__loongarch__) +# define PC_sig(p) EPC_sig(p) +# define FP_sig(p) RFP_sig(p) +# define SP_sig(p) R03_sig(p) +# define LR_sig(p) RRA_sig(p) +# elif defined(__riscv) +# define PC_sig(p) RPC_sig(p) +# define FP_sig(p) RFP_sig(p) +# define SP_sig(p) R02_sig(p) +# define LR_sig(p) RRA_sig(p) +# endif + +static void SetContextPC(CONTEXT* context, uint8_t* pc) { +# ifdef PC_sig + *reinterpret_cast(&PC_sig(context)) = pc; +# else + MOZ_CRASH(); +# endif +} + +static uint8_t* ContextToPC(CONTEXT* context) { +# ifdef PC_sig + return reinterpret_cast(PC_sig(context)); +# else + MOZ_CRASH(); +# endif +} + +static uint8_t* ContextToFP(CONTEXT* context) { +# ifdef FP_sig + return reinterpret_cast(FP_sig(context)); +# else + MOZ_CRASH(); +# endif +} + +static uint8_t* ContextToSP(CONTEXT* context) { +# ifdef SP_sig + return reinterpret_cast(SP_sig(context)); +# else + MOZ_CRASH(); +# endif +} + +# if defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__loongarch__) || defined(__riscv) +static uint8_t* ContextToLR(CONTEXT* context) { +# ifdef LR_sig + return reinterpret_cast(LR_sig(context)); +# else + MOZ_CRASH(); +# endif +} +# endif + +static JS::ProfilingFrameIterator::RegisterState ToRegisterState( + CONTEXT* context) { + JS::ProfilingFrameIterator::RegisterState state; + state.fp = ContextToFP(context); + state.pc = ContextToPC(context); + state.sp = ContextToSP(context); +# if defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__loongarch__) || defined(__riscv) + state.lr = ContextToLR(context); +# else + state.lr = (void*)UINTPTR_MAX; +# endif + return state; +} + +// ============================================================================= +// All signals/exceptions funnel down to this one trap-handling function which +// tests whether the pc is in a wasm module and, if so, whether there is +// actually a trap expected at this pc. These tests both avoid real bugs being +// silently converted to wasm traps and provides the trapping wasm bytecode +// offset we need to report in the error. +// +// Crashing inside wasm trap handling (due to a bug in trap handling or exposed +// during trap handling) must be reported like a normal crash, not cause the +// crash report to be lost. On Windows and non-Mach Unix, a crash during the +// handler reenters the handler, possibly repeatedly until exhausting the stack, +// and so we prevent recursion with the thread-local sAlreadyHandlingTrap. On +// Mach, the wasm exception handler has its own thread and is installed only on +// the thread-level debugging ports of JSRuntime threads, so a crash on +// exception handler thread will not recurse; it will bubble up to the +// process-level debugging ports (where Breakpad is installed). +// ============================================================================= + +static MOZ_THREAD_LOCAL(bool) sAlreadyHandlingTrap; + +struct AutoHandlingTrap { + AutoHandlingTrap() { + MOZ_ASSERT(!sAlreadyHandlingTrap.get()); + sAlreadyHandlingTrap.set(true); + } + + ~AutoHandlingTrap() { + MOZ_ASSERT(sAlreadyHandlingTrap.get()); + sAlreadyHandlingTrap.set(false); + } +}; + +[[nodiscard]] static bool HandleTrap(CONTEXT* context, + JSContext* assertCx = nullptr) { + MOZ_ASSERT(sAlreadyHandlingTrap.get()); + + uint8_t* pc = ContextToPC(context); + const CodeSegment* codeSegment = LookupCodeSegment(pc); + if (!codeSegment || !codeSegment->isModule()) { + return false; + } + + const ModuleSegment& segment = *codeSegment->asModule(); + + Trap trap; + BytecodeOffset bytecode; + if (!segment.code().lookupTrap(pc, &trap, &bytecode)) { + return false; + } + + // We have a safe, expected wasm trap, so fp is well-defined to be a Frame*. + // For the first sanity check, the Trap::IndirectCallBadSig special case is + // due to this trap occurring in the indirect call prologue, while fp points + // to the caller's Frame which can be in a different Module. In any case, + // though, the containing JSContext is the same. + + auto* frame = reinterpret_cast(ContextToFP(context)); + Instance* instance = GetNearestEffectiveInstance(frame); + MOZ_RELEASE_ASSERT(&instance->code() == &segment.code() || + trap == Trap::IndirectCallBadSig); + + JSContext* cx = + instance->realm()->runtimeFromAnyThread()->mainContextFromAnyThread(); + MOZ_RELEASE_ASSERT(!assertCx || cx == assertCx); + + // JitActivation::startWasmTrap() stores enough register state from the + // point of the trap to allow stack unwinding or resumption, both of which + // will call finishWasmTrap(). + jit::JitActivation* activation = cx->activation()->asJit(); + activation->startWasmTrap(trap, bytecode.offset(), ToRegisterState(context)); + SetContextPC(context, segment.trapCode()); + return true; +} + +// ============================================================================= +// The following platform-specific handlers funnel all signals/exceptions into +// the shared HandleTrap() above. +// ============================================================================= + +# if defined(XP_WIN) +// Obtained empirically from thread_local codegen on x86/x64/arm64. +// Compiled in all user binaries, so should be stable over time. +static const unsigned sThreadLocalArrayPointerIndex = 11; + +static LONG WINAPI WasmTrapHandler(LPEXCEPTION_POINTERS exception) { + // Make sure TLS is initialized before reading sAlreadyHandlingTrap. + if (!NtCurrentTeb()->Reserved1[sThreadLocalArrayPointerIndex]) { + return EXCEPTION_CONTINUE_SEARCH; + } + + if (sAlreadyHandlingTrap.get()) { + return EXCEPTION_CONTINUE_SEARCH; + } + AutoHandlingTrap aht; + + EXCEPTION_RECORD* record = exception->ExceptionRecord; + if (record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION && + record->ExceptionCode != EXCEPTION_ILLEGAL_INSTRUCTION) { + return EXCEPTION_CONTINUE_SEARCH; + } + + JSContext* cx = TlsContext.get(); // Cold signal handling code + if (!HandleTrap(exception->ContextRecord, cx)) { + return EXCEPTION_CONTINUE_SEARCH; + } + + return EXCEPTION_CONTINUE_EXECUTION; +} + +# elif defined(XP_DARWIN) +// On OSX we are forced to use the lower-level Mach exception mechanism instead +// of Unix signals because breakpad uses Mach exceptions and would otherwise +// report a crash before wasm gets a chance to handle the exception. + +// This definition was generated by mig (the Mach Interface Generator) for the +// routine 'exception_raise' (exc.defs). +# pragma pack(4) +typedef struct { + mach_msg_header_t Head; + /* start of the kernel processed data */ + mach_msg_body_t msgh_body; + mach_msg_port_descriptor_t thread; + mach_msg_port_descriptor_t task; + /* end of the kernel processed data */ + NDR_record_t NDR; + exception_type_t exception; + mach_msg_type_number_t codeCnt; + int64_t code[2]; +} Request__mach_exception_raise_t; +# pragma pack() + +// The full Mach message also includes a trailer. +struct ExceptionRequest { + Request__mach_exception_raise_t body; + mach_msg_trailer_t trailer; +}; + +static bool HandleMachException(const ExceptionRequest& request) { + // Get the port of the JSContext's thread from the message. + mach_port_t cxThread = request.body.thread.name; + + // Read out the JSRuntime thread's register state. + CONTEXT context; +# if defined(__x86_64__) + unsigned int thread_state_count = x86_THREAD_STATE64_COUNT; + unsigned int float_state_count = x86_FLOAT_STATE64_COUNT; + int thread_state = x86_THREAD_STATE64; + int float_state = x86_FLOAT_STATE64; +# elif defined(__i386__) + unsigned int thread_state_count = x86_THREAD_STATE_COUNT; + unsigned int float_state_count = x86_FLOAT_STATE_COUNT; + int thread_state = x86_THREAD_STATE; + int float_state = x86_FLOAT_STATE; +# elif defined(__arm__) + unsigned int thread_state_count = ARM_THREAD_STATE_COUNT; + unsigned int float_state_count = ARM_NEON_STATE_COUNT; + int thread_state = ARM_THREAD_STATE; + int float_state = ARM_NEON_STATE; +# elif defined(__aarch64__) + unsigned int thread_state_count = ARM_THREAD_STATE64_COUNT; + unsigned int float_state_count = ARM_NEON_STATE64_COUNT; + int thread_state = ARM_THREAD_STATE64; + int float_state = ARM_NEON_STATE64; +# else +# error Unsupported architecture +# endif + kern_return_t kret; + kret = thread_get_state(cxThread, thread_state, + (thread_state_t)&context.thread, &thread_state_count); + if (kret != KERN_SUCCESS) { + return false; + } + kret = thread_get_state(cxThread, float_state, + (thread_state_t)&context.float_, &float_state_count); + if (kret != KERN_SUCCESS) { + return false; + } + + if (request.body.exception != EXC_BAD_ACCESS && + request.body.exception != EXC_BAD_INSTRUCTION) { + return false; + } + + { + AutoNoteSingleThreadedRegion anstr; + AutoHandlingTrap aht; + if (!HandleTrap(&context)) { + return false; + } + } + + // Update the thread state with the new pc and register values. + kret = thread_set_state(cxThread, float_state, + (thread_state_t)&context.float_, float_state_count); + if (kret != KERN_SUCCESS) { + return false; + } + kret = thread_set_state(cxThread, thread_state, + (thread_state_t)&context.thread, thread_state_count); + if (kret != KERN_SUCCESS) { + return false; + } + + return true; +} + +static mach_port_t sMachDebugPort = MACH_PORT_NULL; + +static void MachExceptionHandlerThread() { + ThisThread::SetName("JS Wasm MachExceptionHandler"); + + // Taken from mach_exc in /usr/include/mach/mach_exc.defs. + static const unsigned EXCEPTION_MSG_ID = 2405; + + while (true) { + ExceptionRequest request; + kern_return_t kret = + mach_msg(&request.body.Head, MACH_RCV_MSG, 0, sizeof(request), + sMachDebugPort, MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL); + + // If we fail even receiving the message, we can't even send a reply! + // Rather than hanging the faulting thread (hanging the browser), crash. + if (kret != KERN_SUCCESS) { + fprintf(stderr, "MachExceptionHandlerThread: mach_msg failed with %d\n", + (int)kret); + MOZ_CRASH(); + } + + if (request.body.Head.msgh_id != EXCEPTION_MSG_ID) { + fprintf(stderr, "Unexpected msg header id %d\n", + (int)request.body.Head.msgh_bits); + MOZ_CRASH(); + } + + // Some thread just commited an EXC_BAD_ACCESS and has been suspended by + // the kernel. The kernel is waiting for us to reply with instructions. + // Our default is the "not handled" reply (by setting the RetCode field + // of the reply to KERN_FAILURE) which tells the kernel to continue + // searching at the process and system level. If this is an asm.js + // expected exception, we handle it and return KERN_SUCCESS. + bool handled = HandleMachException(request); + kern_return_t replyCode = handled ? KERN_SUCCESS : KERN_FAILURE; + + // This magic incantation to send a reply back to the kernel was + // derived from the exc_server generated by + // 'mig -v /usr/include/mach/mach_exc.defs'. + __Reply__exception_raise_t reply; + reply.Head.msgh_bits = + MACH_MSGH_BITS(MACH_MSGH_BITS_REMOTE(request.body.Head.msgh_bits), 0); + reply.Head.msgh_size = sizeof(reply); + reply.Head.msgh_remote_port = request.body.Head.msgh_remote_port; + reply.Head.msgh_local_port = MACH_PORT_NULL; + reply.Head.msgh_id = request.body.Head.msgh_id + 100; + reply.NDR = NDR_record; + reply.RetCode = replyCode; + mach_msg(&reply.Head, MACH_SEND_MSG, sizeof(reply), 0, MACH_PORT_NULL, + MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL); + } +} + +# else // If not Windows or Mac, assume Unix + +# if defined(__mips__) || defined(__loongarch__) +static const uint32_t kWasmTrapSignal = SIGFPE; +# else +static const uint32_t kWasmTrapSignal = SIGILL; +# endif + +static struct sigaction sPrevSEGVHandler; +static struct sigaction sPrevSIGBUSHandler; +static struct sigaction sPrevWasmTrapHandler; + +static void WasmTrapHandler(int signum, siginfo_t* info, void* context) { + if (!sAlreadyHandlingTrap.get()) { + AutoHandlingTrap aht; + MOZ_RELEASE_ASSERT(signum == SIGSEGV || signum == SIGBUS || + signum == kWasmTrapSignal); + JSContext* cx = TlsContext.get(); // Cold signal handling code + if (HandleTrap((CONTEXT*)context, cx)) { + return; + } + } + + struct sigaction* previousSignal = nullptr; + switch (signum) { + case SIGSEGV: + previousSignal = &sPrevSEGVHandler; + break; + case SIGBUS: + previousSignal = &sPrevSIGBUSHandler; + break; + case kWasmTrapSignal: + previousSignal = &sPrevWasmTrapHandler; + break; + } + MOZ_ASSERT(previousSignal); + + // This signal is not for any asm.js code we expect, so we need to forward + // the signal to the next handler. If there is no next handler (SIG_IGN or + // SIG_DFL), then it's time to crash. To do this, we set the signal back to + // its original disposition and return. This will cause the faulting op to + // be re-executed which will crash in the normal way. The advantage of + // doing this to calling _exit() is that we remove ourselves from the crash + // stack which improves crash reports. If there is a next handler, call it. + // It will either crash synchronously, fix up the instruction so that + // execution can continue and return, or trigger a crash by returning the + // signal to it's original disposition and returning. + // + // Note: the order of these tests matter. + if (previousSignal->sa_flags & SA_SIGINFO) { + previousSignal->sa_sigaction(signum, info, context); + } else if (previousSignal->sa_handler == SIG_DFL || + previousSignal->sa_handler == SIG_IGN) { + sigaction(signum, previousSignal, nullptr); + } else { + previousSignal->sa_handler(signum); + } +} +# endif // XP_WIN || XP_DARWIN || assume unix + +# if defined(ANDROID) && defined(MOZ_LINKER) +extern "C" MFBT_API bool IsSignalHandlingBroken(); +# endif + +struct InstallState { + bool tried; + bool success; + InstallState() : tried(false), success(false) {} +}; + +static ExclusiveData sEagerInstallState( + mutexid::WasmSignalInstallState); + +#endif // !(JS_CODEGEN_NONE) + +void wasm::EnsureEagerProcessSignalHandlers() { +#ifdef JS_CODEGEN_NONE + // If there is no JIT, then there should be no Wasm signal handlers. + return; +#else + auto eagerInstallState = sEagerInstallState.lock(); + if (eagerInstallState->tried) { + return; + } + + eagerInstallState->tried = true; + MOZ_RELEASE_ASSERT(eagerInstallState->success == false); + +# if defined(ANDROID) && defined(MOZ_LINKER) + // Signal handling is broken on some android systems. + if (IsSignalHandlingBroken()) { + return; + } +# endif + + sAlreadyHandlingTrap.infallibleInit(); + + // Install whatever exception/signal handler is appropriate for the OS. +# if defined(XP_WIN) + +# if defined(MOZ_ASAN) + // Under ASan we need to let the ASan runtime's ShadowExceptionHandler stay + // in the first handler position. + const bool firstHandler = false; +# else + // Otherwise, WasmTrapHandler needs to go first, so that we can recover + // from wasm faults and continue execution without triggering handlers + // such as Breakpad that assume we are crashing. + const bool firstHandler = true; +# endif + if (!AddVectoredExceptionHandler(firstHandler, WasmTrapHandler)) { + // Windows has all sorts of random security knobs for disabling things + // so make this a dynamic failure that disables wasm, not a MOZ_CRASH(). + return; + } + +# elif defined(XP_DARWIN) + // All the Mach setup in EnsureLazyProcessSignalHandlers. +# else + // SA_NODEFER allows us to reenter the signal handler if we crash while + // handling the signal, and fall through to the Breakpad handler by testing + // handlingSegFault. + + // Allow handling OOB with signals on all architectures + struct sigaction faultHandler; + faultHandler.sa_flags = SA_SIGINFO | SA_NODEFER | SA_ONSTACK; + faultHandler.sa_sigaction = WasmTrapHandler; + sigemptyset(&faultHandler.sa_mask); + if (sigaction(SIGSEGV, &faultHandler, &sPrevSEGVHandler)) { + MOZ_CRASH("unable to install segv handler"); + } + +# if defined(JS_CODEGEN_ARM) + // On Arm Handle Unaligned Accesses + struct sigaction busHandler; + busHandler.sa_flags = SA_SIGINFO | SA_NODEFER | SA_ONSTACK; + busHandler.sa_sigaction = WasmTrapHandler; + sigemptyset(&busHandler.sa_mask); + if (sigaction(SIGBUS, &busHandler, &sPrevSIGBUSHandler)) { + MOZ_CRASH("unable to install sigbus handler"); + } +# endif + + // Install a handler to handle the instructions that are emitted to implement + // wasm traps. + struct sigaction wasmTrapHandler; + wasmTrapHandler.sa_flags = SA_SIGINFO | SA_NODEFER | SA_ONSTACK; + wasmTrapHandler.sa_sigaction = WasmTrapHandler; + sigemptyset(&wasmTrapHandler.sa_mask); + if (sigaction(kWasmTrapSignal, &wasmTrapHandler, &sPrevWasmTrapHandler)) { + MOZ_CRASH("unable to install wasm trap handler"); + } +# endif + + eagerInstallState->success = true; +#endif +} + +#ifndef JS_CODEGEN_NONE +static ExclusiveData sLazyInstallState( + mutexid::WasmSignalInstallState); + +static bool EnsureLazyProcessSignalHandlers() { + auto lazyInstallState = sLazyInstallState.lock(); + if (lazyInstallState->tried) { + return lazyInstallState->success; + } + + lazyInstallState->tried = true; + MOZ_RELEASE_ASSERT(lazyInstallState->success == false); + +# ifdef XP_DARWIN + // Create the port that all JSContext threads will redirect their traps to. + kern_return_t kret; + kret = mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, + &sMachDebugPort); + if (kret != KERN_SUCCESS) { + return false; + } + kret = mach_port_insert_right(mach_task_self(), sMachDebugPort, + sMachDebugPort, MACH_MSG_TYPE_MAKE_SEND); + if (kret != KERN_SUCCESS) { + return false; + } + + // Create the thread that will wait on and service sMachDebugPort. + // It's not useful to destroy this thread on process shutdown so + // immediately detach on successful start. + Thread handlerThread; + if (!handlerThread.init(MachExceptionHandlerThread)) { + return false; + } + handlerThread.detach(); +# endif + + lazyInstallState->success = true; + return true; +} +#endif // JS_CODEGEN_NONE + +bool wasm::EnsureFullSignalHandlers(JSContext* cx) { +#ifdef JS_CODEGEN_NONE + return false; +#else + if (cx->wasm().triedToInstallSignalHandlers) { + return cx->wasm().haveSignalHandlers; + } + + cx->wasm().triedToInstallSignalHandlers = true; + MOZ_RELEASE_ASSERT(!cx->wasm().haveSignalHandlers); + + { + auto eagerInstallState = sEagerInstallState.lock(); + MOZ_RELEASE_ASSERT(eagerInstallState->tried); + if (!eagerInstallState->success) { + return false; + } + } + + if (!EnsureLazyProcessSignalHandlers()) { + return false; + } + +# ifdef XP_DARWIN + // In addition to the process-wide signal handler setup, OSX needs each + // thread configured to send its exceptions to sMachDebugPort. While there + // are also task-level (i.e. process-level) exception ports, those are + // "claimed" by breakpad and chaining Mach exceptions is dark magic that we + // avoid by instead intercepting exceptions at the thread level before they + // propagate to the process-level. This works because there are no other + // uses of thread-level exception ports. + MOZ_RELEASE_ASSERT(sMachDebugPort != MACH_PORT_NULL); + thread_port_t thisThread = mach_thread_self(); + kern_return_t kret = thread_set_exception_ports( + thisThread, EXC_MASK_BAD_ACCESS | EXC_MASK_BAD_INSTRUCTION, + sMachDebugPort, EXCEPTION_DEFAULT | MACH_EXCEPTION_CODES, + THREAD_STATE_NONE); + mach_port_deallocate(mach_task_self(), thisThread); + if (kret != KERN_SUCCESS) { + return false; + } +# endif + + cx->wasm().haveSignalHandlers = true; + return true; +#endif +} + +bool wasm::MemoryAccessTraps(const RegisterState& regs, uint8_t* addr, + uint32_t numBytes, uint8_t** newPC) { +#ifdef JS_CODEGEN_NONE + return false; +#else + const wasm::CodeSegment* codeSegment = wasm::LookupCodeSegment(regs.pc); + if (!codeSegment || !codeSegment->isModule()) { + return false; + } + + const wasm::ModuleSegment& segment = *codeSegment->asModule(); + + Trap trap; + BytecodeOffset bytecode; + if (!segment.code().lookupTrap(regs.pc, &trap, &bytecode)) { + return false; + } + switch (trap) { + case Trap::OutOfBounds: + break; + case Trap::NullPointerDereference: + break; +# ifdef WASM_HAS_HEAPREG + case Trap::IndirectCallToNull: + // We use the null pointer exception from loading the heapreg to + // handle indirect calls to null. + break; +# endif + default: + return false; + } + + const Instance& instance = + *GetNearestEffectiveInstance(Frame::fromUntaggedWasmExitFP(regs.fp)); + MOZ_ASSERT(&instance.code() == &segment.code()); + + switch (trap) { + case Trap::OutOfBounds: + if (!instance.memoryAccessInGuardRegion((uint8_t*)addr, numBytes)) { + return false; + } + break; + case Trap::NullPointerDereference: + if ((uintptr_t)addr >= NullPtrGuardSize) { + return false; + } + break; +# ifdef WASM_HAS_HEAPREG + case Trap::IndirectCallToNull: + // Null pointer plus the appropriate offset. + if (addr != + reinterpret_cast(wasm::Instance::offsetOfMemoryBase())) { + return false; + } + break; +# endif + default: + MOZ_CRASH("Should not happen"); + } + + JSContext* cx = TlsContext.get(); // Cold simulator helper function + jit::JitActivation* activation = cx->activation()->asJit(); + activation->startWasmTrap(trap, bytecode.offset(), regs); + *newPC = segment.trapCode(); + return true; +#endif +} + +bool wasm::HandleIllegalInstruction(const RegisterState& regs, + uint8_t** newPC) { +#ifdef JS_CODEGEN_NONE + return false; +#else + const wasm::CodeSegment* codeSegment = wasm::LookupCodeSegment(regs.pc); + if (!codeSegment || !codeSegment->isModule()) { + return false; + } + + const wasm::ModuleSegment& segment = *codeSegment->asModule(); + + Trap trap; + BytecodeOffset bytecode; + if (!segment.code().lookupTrap(regs.pc, &trap, &bytecode)) { + return false; + } + + JSContext* cx = TlsContext.get(); // Cold simulator helper function + jit::JitActivation* activation = cx->activation()->asJit(); + activation->startWasmTrap(trap, bytecode.offset(), regs); + *newPC = segment.trapCode(); + return true; +#endif +} diff --git a/js/src/wasm/WasmSignalHandlers.h b/js/src/wasm/WasmSignalHandlers.h new file mode 100644 index 0000000000..4b452d6b37 --- /dev/null +++ b/js/src/wasm/WasmSignalHandlers.h @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2014 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_signal_handlers_h +#define wasm_signal_handlers_h + +#include "js/ProfilingFrameIterator.h" +#include "wasm/WasmProcess.h" + +namespace js { +namespace wasm { + +using RegisterState = JS::ProfilingFrameIterator::RegisterState; + +// This function performs the low-overhead signal handler initialization that we +// want to do eagerly to ensure a more-deterministic global process state. This +// is especially relevant for signal handlers since handler ordering depends on +// installation order: the wasm signal handler must run *before* the other crash +// handlers (breakpad) and since POSIX signal handlers work LIFO, this function +// needs to be called at the end of the startup process, after the other two +// handlers have been installed. Currently, this is achieved by having +// JSRuntime() call this function. There can be multiple JSRuntimes per process +// so this function can thus be called multiple times, having no effect after +// the first call. +void EnsureEagerProcessSignalHandlers(); + +// Assuming EnsureEagerProcessSignalHandlers() has already been called, +// this function performs the full installation of signal handlers which must +// be performed per-thread/JSContext. This operation may incur some overhead and +// so should be done only when needed to use wasm. Currently, this is done in +// wasm::HasPlatformSupport() which is called when deciding whether to expose +// the 'WebAssembly' object on the global object. +bool EnsureFullSignalHandlers(JSContext* cx); + +// Return whether, with the given simulator register state, a memory access to +// 'addr' of size 'numBytes' needs to trap and, if so, where the simulator +// should redirect pc to. +bool MemoryAccessTraps(const RegisterState& regs, uint8_t* addr, + uint32_t numBytes, uint8_t** newPC); + +// Return whether, with the given simulator register state, an illegal +// instruction fault is expected and, if so, the value of the next PC. +bool HandleIllegalInstruction(const RegisterState& regs, uint8_t** newPC); + +} // namespace wasm +} // namespace js + +#endif // wasm_signal_handlers_h diff --git a/js/src/wasm/WasmStubs.cpp b/js/src/wasm/WasmStubs.cpp new file mode 100644 index 0000000000..7fc61381b9 --- /dev/null +++ b/js/src/wasm/WasmStubs.cpp @@ -0,0 +1,3117 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmStubs.h" + +#include +#include +#include + +#include "jit/ABIArgGenerator.h" +#include "jit/JitFrames.h" +#include "jit/RegisterAllocator.h" +#include "js/Printf.h" +#include "util/Memory.h" +#include "wasm/WasmCode.h" +#include "wasm/WasmGenerator.h" +#include "wasm/WasmInstance.h" + +#include "jit/MacroAssembler-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using MIRTypeVector = Vector; +using ABIArgMIRTypeIter = jit::ABIArgIter; + +/*****************************************************************************/ +// ABIResultIter implementation + +static uint32_t ResultStackSize(ValType type) { + switch (type.kind()) { + case ValType::I32: + return ABIResult::StackSizeOfInt32; + case ValType::I64: + return ABIResult::StackSizeOfInt64; + case ValType::F32: + return ABIResult::StackSizeOfFloat; + case ValType::F64: + return ABIResult::StackSizeOfDouble; +#ifdef ENABLE_WASM_SIMD + case ValType::V128: + return ABIResult::StackSizeOfV128; +#endif + case ValType::Ref: + return ABIResult::StackSizeOfPtr; + default: + MOZ_CRASH("Unexpected result type"); + } +} + +// Compute the size of the stack slot that the wasm ABI requires be allocated +// for a particular MIRType. Note that this sometimes differs from the +// MIRType's natural size. See also ResultStackSize above and ABIResult::size() +// and ABIResultIter below. + +uint32_t js::wasm::MIRTypeToABIResultSize(jit::MIRType type) { + switch (type) { + case MIRType::Int32: + return ABIResult::StackSizeOfInt32; + case MIRType::Int64: + return ABIResult::StackSizeOfInt64; + case MIRType::Float32: + return ABIResult::StackSizeOfFloat; + case MIRType::Double: + return ABIResult::StackSizeOfDouble; +#ifdef ENABLE_WASM_SIMD + case MIRType::Simd128: + return ABIResult::StackSizeOfV128; +#endif + case MIRType::Pointer: + case MIRType::RefOrNull: + return ABIResult::StackSizeOfPtr; + default: + MOZ_CRASH("MIRTypeToABIResultSize - unhandled case"); + } +} + +uint32_t ABIResult::size() const { return ResultStackSize(type()); } + +void ABIResultIter::settleRegister(ValType type) { + MOZ_ASSERT(!done()); + MOZ_ASSERT_IF(direction_ == Next, index() < MaxRegisterResults); + MOZ_ASSERT_IF(direction_ == Prev, index() >= count_ - MaxRegisterResults); + static_assert(MaxRegisterResults == 1, "expected a single register result"); + + switch (type.kind()) { + case ValType::I32: + cur_ = ABIResult(type, ReturnReg); + break; + case ValType::I64: + cur_ = ABIResult(type, ReturnReg64); + break; + case ValType::F32: + cur_ = ABIResult(type, ReturnFloat32Reg); + break; + case ValType::F64: + cur_ = ABIResult(type, ReturnDoubleReg); + break; + case ValType::Ref: + cur_ = ABIResult(type, ReturnReg); + break; +#ifdef ENABLE_WASM_SIMD + case ValType::V128: + cur_ = ABIResult(type, ReturnSimd128Reg); + break; +#endif + default: + MOZ_CRASH("Unexpected result type"); + } +} + +void ABIResultIter::settleNext() { + MOZ_ASSERT(direction_ == Next); + MOZ_ASSERT(!done()); + + uint32_t typeIndex = count_ - index_ - 1; + ValType type = type_[typeIndex]; + + if (index_ < MaxRegisterResults) { + settleRegister(type); + return; + } + + cur_ = ABIResult(type, nextStackOffset_); + nextStackOffset_ += ResultStackSize(type); +} + +void ABIResultIter::settlePrev() { + MOZ_ASSERT(direction_ == Prev); + MOZ_ASSERT(!done()); + uint32_t typeIndex = index_; + ValType type = type_[typeIndex]; + + if (count_ - index_ - 1 < MaxRegisterResults) { + settleRegister(type); + return; + } + + uint32_t size = ResultStackSize(type); + MOZ_ASSERT(nextStackOffset_ >= size); + nextStackOffset_ -= size; + cur_ = ABIResult(type, nextStackOffset_); +} + +#ifdef WASM_CODEGEN_DEBUG +template +static void GenPrint(DebugChannel channel, MacroAssembler& masm, + const Maybe& taken, Closure passArgAndCall) { + if (!IsCodegenDebugEnabled(channel)) { + return; + } + + AllocatableRegisterSet regs(RegisterSet::All()); + LiveRegisterSet save(regs.asLiveSet()); + masm.PushRegsInMask(save); + + if (taken) { + regs.take(taken.value()); + } + Register temp = regs.takeAnyGeneral(); + + { + MOZ_ASSERT(MaybeGetJitContext(), + "codegen debug checks require a jit context"); + masm.setupUnalignedABICall(temp); + passArgAndCall(IsCompilingWasm(), temp); + } + + masm.PopRegsInMask(save); +} + +static void GenPrintf(DebugChannel channel, MacroAssembler& masm, + const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + UniqueChars str = JS_vsmprintf(fmt, ap); + va_end(ap); + + GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) { + // If we've gone this far, it means we're actually using the debugging + // strings. In this case, we leak them! This is only for debugging, and + // doing the right thing is cumbersome (in Ion, it'd mean add a vec of + // strings to the IonScript; in wasm, it'd mean add it to the current + // Module and serialize it properly). + const char* text = str.release(); + + masm.movePtr(ImmPtr((void*)text, ImmPtr::NoCheckToken()), temp); + masm.passABIArg(temp); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintText); + } else { + using Fn = void (*)(const char* output); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + }); +} + +static void GenPrintIsize(DebugChannel channel, MacroAssembler& masm, + const Register& src) { + GenPrint(channel, masm, Some(src), [&](bool inWasm, Register _temp) { + masm.passABIArg(src); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintI32); + } else { + using Fn = void (*)(int32_t val); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + }); +} + +static void GenPrintPtr(DebugChannel channel, MacroAssembler& masm, + const Register& src) { + GenPrint(channel, masm, Some(src), [&](bool inWasm, Register _temp) { + masm.passABIArg(src); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintPtr); + } else { + using Fn = void (*)(uint8_t* val); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + }); +} + +static void GenPrintI64(DebugChannel channel, MacroAssembler& masm, + const Register64& src) { +# if JS_BITS_PER_WORD == 64 + GenPrintf(channel, masm, "i64 "); + GenPrintIsize(channel, masm, src.reg); +# else + GenPrintf(channel, masm, "i64("); + GenPrintIsize(channel, masm, src.low); + GenPrintIsize(channel, masm, src.high); + GenPrintf(channel, masm, ") "); +# endif +} + +static void GenPrintF32(DebugChannel channel, MacroAssembler& masm, + const FloatRegister& src) { + GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) { + masm.passABIArg(src, MoveOp::FLOAT32); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintF32); + } else { + using Fn = void (*)(float val); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + }); +} + +static void GenPrintF64(DebugChannel channel, MacroAssembler& masm, + const FloatRegister& src) { + GenPrint(channel, masm, Nothing(), [&](bool inWasm, Register temp) { + masm.passABIArg(src, MoveOp::DOUBLE); + if (inWasm) { + masm.callDebugWithABI(SymbolicAddress::PrintF64); + } else { + using Fn = void (*)(double val); + masm.callWithABI(MoveOp::GENERAL, + CheckUnsafeCallWithABI::DontCheckOther); + } + }); +} + +# ifdef ENABLE_WASM_SIMD +static void GenPrintV128(DebugChannel channel, MacroAssembler& masm, + const FloatRegister& src) { + // TODO: We might try to do something meaningful here once SIMD data are + // aligned and hence C++-ABI compliant. For now, just make ourselves visible. + GenPrintf(channel, masm, "v128"); +} +# endif +#else +static void GenPrintf(DebugChannel channel, MacroAssembler& masm, + const char* fmt, ...) {} +static void GenPrintIsize(DebugChannel channel, MacroAssembler& masm, + const Register& src) {} +static void GenPrintPtr(DebugChannel channel, MacroAssembler& masm, + const Register& src) {} +static void GenPrintI64(DebugChannel channel, MacroAssembler& masm, + const Register64& src) {} +static void GenPrintF32(DebugChannel channel, MacroAssembler& masm, + const FloatRegister& src) {} +static void GenPrintF64(DebugChannel channel, MacroAssembler& masm, + const FloatRegister& src) {} +# ifdef ENABLE_WASM_SIMD +static void GenPrintV128(DebugChannel channel, MacroAssembler& masm, + const FloatRegister& src) {} +# endif +#endif + +static bool FinishOffsets(MacroAssembler& masm, Offsets* offsets) { + // On old ARM hardware, constant pools could be inserted and they need to + // be flushed before considering the size of the masm. + masm.flushBuffer(); + offsets->end = masm.size(); + return !masm.oom(); +} + +static void AssertStackAlignment(MacroAssembler& masm, uint32_t alignment, + uint32_t addBeforeAssert = 0) { + MOZ_ASSERT( + (sizeof(Frame) + masm.framePushed() + addBeforeAssert) % alignment == 0); + masm.assertStackAlignment(alignment, addBeforeAssert); +} + +template class ABIArgIterT> +static unsigned StackArgBytesHelper(const VectorT& args) { + ABIArgIterT iter(args); + while (!iter.done()) { + iter++; + } + return iter.stackBytesConsumedSoFar(); +} + +template +static unsigned StackArgBytesForNativeABI(const VectorT& args) { + return StackArgBytesHelper(args); +} + +template +static unsigned StackArgBytesForWasmABI(const VectorT& args) { + return StackArgBytesHelper(args); +} + +static unsigned StackArgBytesForWasmABI(const FuncType& funcType) { + ArgTypeVector args(funcType); + return StackArgBytesForWasmABI(args); +} + +static void Move64(MacroAssembler& masm, const Address& src, + const Address& dest, Register scratch) { +#if JS_BITS_PER_WORD == 32 + MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch); + masm.load32(LowWord(src), scratch); + masm.store32(scratch, LowWord(dest)); + masm.load32(HighWord(src), scratch); + masm.store32(scratch, HighWord(dest)); +#else + Register64 scratch64(scratch); + masm.load64(src, scratch64); + masm.store64(scratch64, dest); +#endif +} + +static void SetupABIArguments(MacroAssembler& masm, const FuncExport& fe, + const FuncType& funcType, Register argv, + Register scratch) { + // Copy parameters out of argv and into the registers/stack-slots specified by + // the wasm ABI. + // + // SetupABIArguments are only used for C++ -> wasm calls through callExport(), + // and V128 and Ref types (other than externref) are not currently allowed. + ArgTypeVector args(funcType); + for (WasmABIArgIter iter(args); !iter.done(); iter++) { + unsigned argOffset = iter.index() * sizeof(ExportArg); + Address src(argv, argOffset); + MIRType type = iter.mirType(); + switch (iter->kind()) { + case ABIArg::GPR: + if (type == MIRType::Int32) { + masm.load32(src, iter->gpr()); + } else if (type == MIRType::Int64) { + masm.load64(src, iter->gpr64()); + } else if (type == MIRType::RefOrNull) { + masm.loadPtr(src, iter->gpr()); + } else if (type == MIRType::StackResults) { + MOZ_ASSERT(args.isSyntheticStackResultPointerArg(iter.index())); + masm.loadPtr(src, iter->gpr()); + } else { + MOZ_CRASH("unknown GPR type"); + } + break; +#ifdef JS_CODEGEN_REGISTER_PAIR + case ABIArg::GPR_PAIR: + if (type == MIRType::Int64) { + masm.load64(src, iter->gpr64()); + } else { + MOZ_CRASH("wasm uses hardfp for function calls."); + } + break; +#endif + case ABIArg::FPU: { + static_assert(sizeof(ExportArg) >= jit::Simd128DataSize, + "ExportArg must be big enough to store SIMD values"); + switch (type) { + case MIRType::Double: + masm.loadDouble(src, iter->fpu()); + break; + case MIRType::Float32: + masm.loadFloat32(src, iter->fpu()); + break; + case MIRType::Simd128: +#ifdef ENABLE_WASM_SIMD + // This is only used by the testing invoke path, + // wasmLosslessInvoke, and is guarded against in normal JS-API + // call paths. + masm.loadUnalignedSimd128(src, iter->fpu()); + break; +#else + MOZ_CRASH("V128 not supported in SetupABIArguments"); +#endif + default: + MOZ_CRASH("unexpected FPU type"); + break; + } + break; + } + case ABIArg::Stack: + switch (type) { + case MIRType::Int32: + masm.load32(src, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + iter->offsetFromArgBase())); + break; + case MIRType::Int64: { + RegisterOrSP sp = masm.getStackPointer(); + Move64(masm, src, Address(sp, iter->offsetFromArgBase()), scratch); + break; + } + case MIRType::RefOrNull: + masm.loadPtr(src, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + iter->offsetFromArgBase())); + break; + case MIRType::Double: { + ScratchDoubleScope fpscratch(masm); + masm.loadDouble(src, fpscratch); + masm.storeDouble(fpscratch, Address(masm.getStackPointer(), + iter->offsetFromArgBase())); + break; + } + case MIRType::Float32: { + ScratchFloat32Scope fpscratch(masm); + masm.loadFloat32(src, fpscratch); + masm.storeFloat32(fpscratch, Address(masm.getStackPointer(), + iter->offsetFromArgBase())); + break; + } + case MIRType::Simd128: { +#ifdef ENABLE_WASM_SIMD + // This is only used by the testing invoke path, + // wasmLosslessInvoke, and is guarded against in normal JS-API + // call paths. + ScratchSimd128Scope fpscratch(masm); + masm.loadUnalignedSimd128(src, fpscratch); + masm.storeUnalignedSimd128( + fpscratch, + Address(masm.getStackPointer(), iter->offsetFromArgBase())); + break; +#else + MOZ_CRASH("V128 not supported in SetupABIArguments"); +#endif + } + case MIRType::StackResults: { + MOZ_ASSERT(args.isSyntheticStackResultPointerArg(iter.index())); + masm.loadPtr(src, scratch); + masm.storePtr(scratch, Address(masm.getStackPointer(), + iter->offsetFromArgBase())); + break; + } + default: + MOZ_CRASH("unexpected stack arg type"); + } + break; + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + } +} + +static void StoreRegisterResult(MacroAssembler& masm, const FuncExport& fe, + const FuncType& funcType, Register loc) { + ResultType results = ResultType::Vector(funcType.results()); + DebugOnly sawRegisterResult = false; + for (ABIResultIter iter(results); !iter.done(); iter.next()) { + const ABIResult& result = iter.cur(); + if (result.inRegister()) { + MOZ_ASSERT(!sawRegisterResult); + sawRegisterResult = true; + switch (result.type().kind()) { + case ValType::I32: + masm.store32(result.gpr(), Address(loc, 0)); + break; + case ValType::I64: + masm.store64(result.gpr64(), Address(loc, 0)); + break; + case ValType::V128: +#ifdef ENABLE_WASM_SIMD + masm.storeUnalignedSimd128(result.fpr(), Address(loc, 0)); + break; +#else + MOZ_CRASH("V128 not supported in StoreABIReturn"); +#endif + case ValType::F32: + masm.storeFloat32(result.fpr(), Address(loc, 0)); + break; + case ValType::F64: + masm.storeDouble(result.fpr(), Address(loc, 0)); + break; + case ValType::Ref: + masm.storePtr(result.gpr(), Address(loc, 0)); + break; + } + } + } + MOZ_ASSERT(sawRegisterResult == (results.length() > 0)); +} + +#if defined(JS_CODEGEN_ARM) +// The ARM system ABI also includes d15 & s31 in the non volatile float +// registers. Also exclude lr (a.k.a. r14) as we preserve it manually. +static const LiveRegisterSet NonVolatileRegs = LiveRegisterSet( + GeneralRegisterSet(Registers::NonVolatileMask & + ~(Registers::SetType(1) << Registers::lr)), + FloatRegisterSet(FloatRegisters::NonVolatileMask | + (FloatRegisters::SetType(1) << FloatRegisters::d15) | + (FloatRegisters::SetType(1) << FloatRegisters::s31))); +#elif defined(JS_CODEGEN_ARM64) +// Exclude the Link Register (x30) because it is preserved manually. +// +// Include x16 (scratch) to make a 16-byte aligned amount of integer registers. +// Include d31 (scratch) to make a 16-byte aligned amount of floating registers. +static const LiveRegisterSet NonVolatileRegs = LiveRegisterSet( + GeneralRegisterSet((Registers::NonVolatileMask & + ~(Registers::SetType(1) << Registers::lr)) | + (Registers::SetType(1) << Registers::x16)), + FloatRegisterSet(FloatRegisters::NonVolatileMask | + FloatRegisters::NonAllocatableMask)); +#else +static const LiveRegisterSet NonVolatileRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::NonVolatileMask), + FloatRegisterSet(FloatRegisters::NonVolatileMask)); +#endif + +static const unsigned NumExtraPushed = 2; // instance and argv + +#ifdef JS_CODEGEN_ARM64 +static const unsigned WasmPushSize = 16; +#else +static const unsigned WasmPushSize = sizeof(void*); +#endif + +static void AssertExpectedSP(MacroAssembler& masm) { +#ifdef JS_CODEGEN_ARM64 + MOZ_ASSERT(sp.Is(masm.GetStackPointer64())); +# ifdef DEBUG + // Since we're asserting that SP is the currently active stack pointer, + // let's also in effect assert that PSP is dead -- by setting it to 1, so as + // to cause to cause any attempts to use it to segfault in an easily + // identifiable way. + masm.asVIXL().Mov(PseudoStackPointer64, 1); +# endif +#endif +} + +template +static void WasmPush(MacroAssembler& masm, const Operand& op) { +#ifdef JS_CODEGEN_ARM64 + // Allocate a pad word so that SP can remain properly aligned. |op| will be + // written at the lower-addressed of the two words pushed here. + masm.reserveStack(WasmPushSize); + masm.storePtr(op, Address(masm.getStackPointer(), 0)); +#else + masm.Push(op); +#endif +} + +static void WasmPop(MacroAssembler& masm, Register r) { +#ifdef JS_CODEGEN_ARM64 + // Also pop the pad word allocated by WasmPush. + masm.loadPtr(Address(masm.getStackPointer(), 0), r); + masm.freeStack(WasmPushSize); +#else + masm.Pop(r); +#endif +} + +static void MoveSPForJitABI(MacroAssembler& masm) { +#ifdef JS_CODEGEN_ARM64 + masm.moveStackPtrTo(PseudoStackPointer); +#endif +} + +static void CallFuncExport(MacroAssembler& masm, const FuncExport& fe, + const Maybe& funcPtr) { + MOZ_ASSERT(fe.hasEagerStubs() == !funcPtr); + MoveSPForJitABI(masm); + if (funcPtr) { + masm.call(*funcPtr); + } else { + masm.call(CallSiteDesc(CallSiteDesc::Func), fe.funcIndex()); + } +} + +STATIC_ASSERT_ANYREF_IS_JSOBJECT; // Strings are currently boxed + +// Unboxing is branchy and contorted because of Spectre mitigations - we don't +// have enough scratch registers. Were it not for the spectre mitigations in +// branchTestObjClass, the branch nest below would be restructured significantly +// by inverting branches and using fewer registers. + +// Unbox an anyref in src (clobbering src in the process) and then re-box it as +// a Value in *dst. See the definition of AnyRef for a discussion of pointer +// representation. +static void UnboxAnyrefIntoValue(MacroAssembler& masm, Register instance, + Register src, const Address& dst, + Register scratch) { + MOZ_ASSERT(src != scratch); + + // Not actually the value we're passing, but we've no way of + // decoding anything better. + GenPrintPtr(DebugChannel::Import, masm, src); + + Label notNull, mustUnbox, done; + masm.branchTestPtr(Assembler::NonZero, src, src, ¬Null); + masm.storeValue(NullValue(), dst); + masm.jump(&done); + + masm.bind(¬Null); + // The type test will clear src if the test fails, so store early. + masm.storeValue(JSVAL_TYPE_OBJECT, src, dst); + // Spectre mitigations: see comment above about efficiency. + masm.branchTestObjClass(Assembler::Equal, src, + Address(instance, Instance::offsetOfValueBoxClass()), + scratch, src, &mustUnbox); + masm.jump(&done); + + masm.bind(&mustUnbox); + Move64(masm, Address(src, WasmValueBox::offsetOfValue()), dst, scratch); + + masm.bind(&done); +} + +// Unbox an anyref in src and then re-box it as a Value in dst. +// See the definition of AnyRef for a discussion of pointer representation. +static void UnboxAnyrefIntoValueReg(MacroAssembler& masm, Register instance, + Register src, ValueOperand dst, + Register scratch) { + MOZ_ASSERT(src != scratch); +#if JS_BITS_PER_WORD == 32 + MOZ_ASSERT(dst.typeReg() != scratch); + MOZ_ASSERT(dst.payloadReg() != scratch); +#else + MOZ_ASSERT(dst.valueReg() != scratch); +#endif + + // Not actually the value we're passing, but we've no way of + // decoding anything better. + GenPrintPtr(DebugChannel::Import, masm, src); + + Label notNull, mustUnbox, done; + masm.branchTestPtr(Assembler::NonZero, src, src, ¬Null); + masm.moveValue(NullValue(), dst); + masm.jump(&done); + + masm.bind(¬Null); + // The type test will clear src if the test fails, so store early. + masm.moveValue(TypedOrValueRegister(MIRType::Object, AnyRegister(src)), dst); + // Spectre mitigations: see comment above about efficiency. + masm.branchTestObjClass(Assembler::Equal, src, + Address(instance, Instance::offsetOfValueBoxClass()), + scratch, src, &mustUnbox); + masm.jump(&done); + + masm.bind(&mustUnbox); + masm.loadValue(Address(src, WasmValueBox::offsetOfValue()), dst); + + masm.bind(&done); +} + +// Box the Value in src as an anyref in dest. src and dest must not overlap. +// See the definition of AnyRef for a discussion of pointer representation. +static void BoxValueIntoAnyref(MacroAssembler& masm, ValueOperand src, + Register dest, Label* oolConvert) { + Label nullValue, objectValue, done; + { + ScratchTagScope tag(masm, src); + masm.splitTagForTest(src, tag); + masm.branchTestObject(Assembler::Equal, tag, &objectValue); + masm.branchTestNull(Assembler::Equal, tag, &nullValue); + masm.jump(oolConvert); + } + + masm.bind(&nullValue); + masm.xorPtr(dest, dest); + masm.jump(&done); + + masm.bind(&objectValue); + masm.unboxObject(src, dest); + + masm.bind(&done); +} + +// Generate a stub that enters wasm from a C++ caller via the native ABI. The +// signature of the entry point is Module::ExportFuncPtr. The exported wasm +// function has an ABI derived from its specific signature, so this function +// must map from the ABI of ExportFuncPtr to the export's signature's ABI. +static bool GenerateInterpEntry(MacroAssembler& masm, const FuncExport& fe, + const FuncType& funcType, + const Maybe& funcPtr, + Offsets* offsets) { + AutoCreatedBy acb(masm, "GenerateInterpEntry"); + + AssertExpectedSP(masm); + masm.haltingAlign(CodeAlignment); + + offsets->begin = masm.currentOffset(); + + // Save the return address if it wasn't already saved by the call insn. +#ifdef JS_USE_LINK_REGISTER +# if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) + masm.pushReturnAddress(); +# elif defined(JS_CODEGEN_ARM64) + // WasmPush updates framePushed() unlike pushReturnAddress(), but that's + // cancelled by the setFramePushed() below. + WasmPush(masm, lr); +# else + MOZ_CRASH("Implement this"); +# endif +#endif + + // Save all caller non-volatile registers before we clobber them here and in + // the wasm callee (which does not preserve non-volatile registers). + masm.setFramePushed(0); + masm.PushRegsInMask(NonVolatileRegs); + + const unsigned nonVolatileRegsPushSize = + masm.PushRegsInMaskSizeInBytes(NonVolatileRegs); + + MOZ_ASSERT(masm.framePushed() == nonVolatileRegsPushSize); + + // Put the 'argv' argument into a non-argument/return/instance register so + // that we can use 'argv' while we fill in the arguments for the wasm callee. + // Use a second non-argument/return register as temporary scratch. + Register argv = ABINonArgReturnReg0; + Register scratch = ABINonArgReturnReg1; + + // scratch := SP + masm.moveStackPtrTo(scratch); + + // Dynamically align the stack since ABIStackAlignment is not necessarily + // WasmStackAlignment. Preserve SP so it can be restored after the call. +#ifdef JS_CODEGEN_ARM64 + static_assert(WasmStackAlignment == 16, "ARM64 SP alignment"); +#else + masm.andToStackPtr(Imm32(~(WasmStackAlignment - 1))); +#endif + masm.assertStackAlignment(WasmStackAlignment); + + // Create a fake frame: just previous RA and an FP. + const size_t FakeFrameSize = 2 * sizeof(void*); +#ifdef JS_CODEGEN_ARM64 + masm.Ldr(ARMRegister(ABINonArgReturnReg0, 64), + MemOperand(ARMRegister(scratch, 64), nonVolatileRegsPushSize)); +#else + masm.Push(Address(scratch, nonVolatileRegsPushSize)); +#endif + // Store fake wasm register state. Ensure the frame pointer passed by the C++ + // caller doesn't have the ExitFPTag bit set to not confuse frame iterators. + // This bit shouldn't be set if C++ code is using frame pointers, so this has + // no effect on native stack unwinders. + masm.andPtr(Imm32(int32_t(~ExitFPTag)), FramePointer); +#ifdef JS_CODEGEN_ARM64 + masm.asVIXL().Push(ARMRegister(ABINonArgReturnReg0, 64), + ARMRegister(FramePointer, 64)); + masm.moveStackPtrTo(FramePointer); +#else + masm.Push(FramePointer); +#endif + + masm.moveStackPtrTo(FramePointer); + masm.setFramePushed(FakeFrameSize); +#ifdef JS_CODEGEN_ARM64 + const size_t FakeFramePushed = 0; +#else + const size_t FakeFramePushed = sizeof(void*); + masm.Push(scratch); +#endif + + // Read the arguments of wasm::ExportFuncPtr according to the native ABI. + // The entry stub's frame is 1 word. + const unsigned argBase = sizeof(void*) + nonVolatileRegsPushSize; + ABIArgGenerator abi; + ABIArg arg; + + // arg 1: ExportArg* + arg = abi.next(MIRType::Pointer); + if (arg.kind() == ABIArg::GPR) { + masm.movePtr(arg.gpr(), argv); + } else { + masm.loadPtr(Address(scratch, argBase + arg.offsetFromArgBase()), argv); + } + + // Arg 2: Instance* + arg = abi.next(MIRType::Pointer); + if (arg.kind() == ABIArg::GPR) { + masm.movePtr(arg.gpr(), InstanceReg); + } else { + masm.loadPtr(Address(scratch, argBase + arg.offsetFromArgBase()), + InstanceReg); + } + + WasmPush(masm, InstanceReg); + + // Save 'argv' on the stack so that we can recover it after the call. + WasmPush(masm, argv); + + MOZ_ASSERT(masm.framePushed() == + NumExtraPushed * WasmPushSize + FakeFrameSize + FakeFramePushed); + + // Reserve stack space for the wasm call. + unsigned argDecrement = + StackDecrementForCall(WasmStackAlignment, masm.framePushed(), + StackArgBytesForWasmABI(funcType)); + masm.reserveStack(argDecrement); + + // Copy parameters out of argv and into the wasm ABI registers/stack-slots. + SetupABIArguments(masm, fe, funcType, argv, scratch); + + masm.loadWasmPinnedRegsFromInstance(); + + masm.storePtr(InstanceReg, Address(masm.getStackPointer(), + WasmCalleeInstanceOffsetBeforeCall)); + + // Call into the real function. Note that, due to the throw stub, fp, instance + // and pinned registers may be clobbered. + masm.assertStackAlignment(WasmStackAlignment); + CallFuncExport(masm, fe, funcPtr); + masm.assertStackAlignment(WasmStackAlignment); + + // Set the return value based on whether InstanceReg is the FailInstanceReg + // magic value (set by the throw stub). + Label success, join; + masm.branchPtr(Assembler::NotEqual, InstanceReg, Imm32(FailInstanceReg), + &success); + masm.move32(Imm32(false), scratch); + masm.jump(&join); + masm.bind(&success); + masm.move32(Imm32(true), scratch); + masm.bind(&join); + + // Pop the arguments pushed after the dynamic alignment. + masm.freeStack(argDecrement); + + masm.setFramePushed(NumExtraPushed * WasmPushSize + FakeFrameSize + + FakeFramePushed); + + // Recover the 'argv' pointer which was saved before aligning the stack. + WasmPop(masm, argv); + + WasmPop(masm, InstanceReg); + + // Pop the stack pointer to its value right before dynamic alignment. +#ifdef JS_CODEGEN_ARM64 + static_assert(WasmStackAlignment == 16, "ARM64 SP alignment"); + masm.freeStack(FakeFrameSize); +#else + masm.PopStackPtr(); +#endif + + // Store the register result, if any, in argv[0]. + // No widening is required, as the value leaves ReturnReg. + StoreRegisterResult(masm, fe, funcType, argv); + + masm.move32(scratch, ReturnReg); + + // Restore clobbered non-volatile registers of the caller. + masm.setFramePushed(nonVolatileRegsPushSize); + masm.PopRegsInMask(NonVolatileRegs); + MOZ_ASSERT(masm.framePushed() == 0); + +#if defined(JS_CODEGEN_ARM64) + masm.setFramePushed(WasmPushSize); + WasmPop(masm, lr); + masm.abiret(); +#else + masm.ret(); +#endif + + return FinishOffsets(masm, offsets); +} + +#ifdef JS_PUNBOX64 +static const ValueOperand ScratchValIonEntry = ValueOperand(ABINonArgReg0); +#else +static const ValueOperand ScratchValIonEntry = + ValueOperand(ABINonArgReg0, ABINonArgReg1); +#endif +static const Register ScratchIonEntry = ABINonArgReg2; + +static void CallSymbolicAddress(MacroAssembler& masm, bool isAbsolute, + SymbolicAddress sym) { + if (isAbsolute) { + masm.call(ImmPtr(SymbolicAddressTarget(sym), ImmPtr::NoCheckToken())); + } else { + masm.call(sym); + } +} + +// Load instance's instance from the callee. +static void GenerateJitEntryLoadInstance(MacroAssembler& masm) { + // ScratchIonEntry := callee => JSFunction* + unsigned offset = JitFrameLayout::offsetOfCalleeToken(); + masm.loadFunctionFromCalleeToken(Address(FramePointer, offset), + ScratchIonEntry); + + // ScratchIonEntry := callee->getExtendedSlot(WASM_INSTANCE_SLOT)->toPrivate() + // => Instance* + offset = FunctionExtended::offsetOfExtendedSlot( + FunctionExtended::WASM_INSTANCE_SLOT); + masm.loadPrivate(Address(ScratchIonEntry, offset), InstanceReg); +} + +// Creates a JS fake exit frame for wasm, so the frame iterators just use +// JSJit frame iteration. +static void GenerateJitEntryThrow(MacroAssembler& masm, unsigned frameSize) { + AssertExpectedSP(masm); + + MOZ_ASSERT(masm.framePushed() == frameSize); + + masm.freeStack(frameSize); + MoveSPForJitABI(masm); + + GenerateJitEntryLoadInstance(masm); + + masm.loadPtr(Address(InstanceReg, Instance::offsetOfCx()), ScratchIonEntry); + masm.enterFakeExitFrameForWasm(ScratchIonEntry, ScratchIonEntry, + ExitFrameType::WasmGenericJitEntry); + + masm.loadPtr(Address(InstanceReg, Instance::offsetOfJSJitExceptionHandler()), + ScratchIonEntry); + masm.jump(ScratchIonEntry); +} + +// Helper function for allocating a BigInt and initializing it from an I64 in +// GenerateJitEntry. The return result is written to scratch. +// +// Note that this will create a new frame and must not - in its current form - +// be called from a context where there is already another stub frame on the +// stack, as that confuses unwinding during profiling. This was a problem for +// its use from GenerateImportJitExit, see bug 1754258. Therefore, +// FuncType::canHaveJitExit prevents the present function from being called for +// exits. +static void GenerateBigIntInitialization(MacroAssembler& masm, + unsigned bytesPushedByPrologue, + Register64 input, Register scratch, + const FuncExport& fe, Label* fail) { +#if JS_BITS_PER_WORD == 32 + MOZ_ASSERT(input.low != scratch); + MOZ_ASSERT(input.high != scratch); +#else + MOZ_ASSERT(input.reg != scratch); +#endif + + // We need to avoid clobbering other argument registers and the input. + AllocatableRegisterSet regs(RegisterSet::Volatile()); + LiveRegisterSet save(regs.asLiveSet()); + masm.PushRegsInMask(save); + + unsigned frameSize = StackDecrementForCall( + ABIStackAlignment, masm.framePushed() + bytesPushedByPrologue, 0); + masm.reserveStack(frameSize); + masm.assertStackAlignment(ABIStackAlignment); + + CallSymbolicAddress(masm, !fe.hasEagerStubs(), + SymbolicAddress::AllocateBigInt); + masm.storeCallPointerResult(scratch); + + masm.assertStackAlignment(ABIStackAlignment); + masm.freeStack(frameSize); + + LiveRegisterSet ignore; + ignore.add(scratch); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.branchTest32(Assembler::Zero, scratch, scratch, fail); + masm.initializeBigInt64(Scalar::BigInt64, scratch, input); +} + +// Generate a stub that enters wasm from a jit code caller via the jit ABI. +// +// ARM64 note: This does not save the PseudoStackPointer so we must be sure to +// recompute it on every return path, be it normal return or exception return. +// The JIT code we return to assumes it is correct. + +static bool GenerateJitEntry(MacroAssembler& masm, size_t funcExportIndex, + const FuncExport& fe, const FuncType& funcType, + const Maybe& funcPtr, + CallableOffsets* offsets) { + AutoCreatedBy acb(masm, "GenerateJitEntry"); + + AssertExpectedSP(masm); + + RegisterOrSP sp = masm.getStackPointer(); + + GenerateJitEntryPrologue(masm, offsets); + + // The jit caller has set up the following stack layout (sp grows to the + // left): + // <-- retAddr | descriptor | callee | argc | this | arg1..N + // + // GenerateJitEntryPrologue has additionally pushed the caller's frame + // pointer. The stack pointer is now JitStackAlignment-aligned. + + MOZ_ASSERT(masm.framePushed() == 0); + + unsigned normalBytesNeeded = StackArgBytesForWasmABI(funcType); + + MIRTypeVector coerceArgTypes; + MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Int32)); + MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer)); + MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer)); + unsigned oolBytesNeeded = StackArgBytesForWasmABI(coerceArgTypes); + + unsigned bytesNeeded = std::max(normalBytesNeeded, oolBytesNeeded); + + // Note the jit caller ensures the stack is aligned *after* the call + // instruction. + unsigned frameSizeExclFP = StackDecrementForCall( + WasmStackAlignment, masm.framePushed(), bytesNeeded); + + // Reserve stack space for wasm ABI arguments, set up like this: + // <-- ABI args | padding + masm.reserveStack(frameSizeExclFP); + + uint32_t frameSize = masm.framePushed(); + + GenerateJitEntryLoadInstance(masm); + + if (funcType.hasUnexposableArgOrRet()) { + CallSymbolicAddress(masm, !fe.hasEagerStubs(), + SymbolicAddress::ReportV128JSCall); + GenerateJitEntryThrow(masm, frameSize); + return FinishOffsets(masm, offsets); + } + + FloatRegister scratchF = ABINonArgDoubleReg; + Register scratchG = ScratchIonEntry; + ValueOperand scratchV = ScratchValIonEntry; + + GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; arguments ", + fe.funcIndex()); + + // We do two loops: + // - one loop up-front will make sure that all the Value tags fit the + // expected signature argument types. If at least one inline conversion + // fails, we just jump to the OOL path which will call into C++. Inline + // conversions are ordered in the way we expect them to happen the most. + // - the second loop will unbox the arguments into the right registers. + Label oolCall; + for (size_t i = 0; i < funcType.args().length(); i++) { + Address jitArgAddr(FramePointer, JitFrameLayout::offsetOfActualArg(i)); + masm.loadValue(jitArgAddr, scratchV); + + Label next; + switch (funcType.args()[i].kind()) { + case ValType::I32: { + ScratchTagScope tag(masm, scratchV); + masm.splitTagForTest(scratchV, tag); + + // For int32 inputs, just skip. + masm.branchTestInt32(Assembler::Equal, tag, &next); + + // For double inputs, unbox, truncate and store back. + Label storeBack, notDouble; + masm.branchTestDouble(Assembler::NotEqual, tag, ¬Double); + { + ScratchTagScopeRelease _(&tag); + masm.unboxDouble(scratchV, scratchF); + masm.branchTruncateDoubleMaybeModUint32(scratchF, scratchG, &oolCall); + masm.jump(&storeBack); + } + masm.bind(¬Double); + + // For null or undefined, store 0. + Label nullOrUndefined, notNullOrUndefined; + masm.branchTestUndefined(Assembler::Equal, tag, &nullOrUndefined); + masm.branchTestNull(Assembler::NotEqual, tag, ¬NullOrUndefined); + masm.bind(&nullOrUndefined); + { + ScratchTagScopeRelease _(&tag); + masm.storeValue(Int32Value(0), jitArgAddr); + } + masm.jump(&next); + masm.bind(¬NullOrUndefined); + + // For booleans, store the number value back. Other types (symbol, + // object, strings) go to the C++ call. + masm.branchTestBoolean(Assembler::NotEqual, tag, &oolCall); + masm.unboxBoolean(scratchV, scratchG); + // fallthrough: + + masm.bind(&storeBack); + { + ScratchTagScopeRelease _(&tag); + masm.storeValue(JSVAL_TYPE_INT32, scratchG, jitArgAddr); + } + break; + } + case ValType::I64: { + ScratchTagScope tag(masm, scratchV); + masm.splitTagForTest(scratchV, tag); + + // For BigInt inputs, just skip. Otherwise go to C++ for other + // types that require creating a new BigInt or erroring. + masm.branchTestBigInt(Assembler::NotEqual, tag, &oolCall); + masm.jump(&next); + break; + } + case ValType::F32: + case ValType::F64: { + // Note we can reuse the same code for f32/f64 here, since for the + // case of f32, the conversion of f64 to f32 will happen in the + // second loop. + ScratchTagScope tag(masm, scratchV); + masm.splitTagForTest(scratchV, tag); + + // For double inputs, just skip. + masm.branchTestDouble(Assembler::Equal, tag, &next); + + // For int32 inputs, convert and rebox. + Label storeBack, notInt32; + { + ScratchTagScopeRelease _(&tag); + masm.branchTestInt32(Assembler::NotEqual, scratchV, ¬Int32); + masm.int32ValueToDouble(scratchV, scratchF); + masm.jump(&storeBack); + } + masm.bind(¬Int32); + + // For undefined (missing argument), store NaN. + Label notUndefined; + masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined); + { + ScratchTagScopeRelease _(&tag); + masm.storeValue(DoubleValue(JS::GenericNaN()), jitArgAddr); + masm.jump(&next); + } + masm.bind(¬Undefined); + + // +null is 0. + Label notNull; + masm.branchTestNull(Assembler::NotEqual, tag, ¬Null); + { + ScratchTagScopeRelease _(&tag); + masm.storeValue(DoubleValue(0.), jitArgAddr); + } + masm.jump(&next); + masm.bind(¬Null); + + // For booleans, store the number value back. Other types (symbol, + // object, strings) go to the C++ call. + masm.branchTestBoolean(Assembler::NotEqual, tag, &oolCall); + masm.boolValueToDouble(scratchV, scratchF); + // fallthrough: + + masm.bind(&storeBack); + { + ScratchTagScopeRelease _(&tag); + masm.boxDouble(scratchF, jitArgAddr); + } + break; + } + case ValType::Ref: { + // Guarded against by temporarilyUnsupportedReftypeForEntry() + MOZ_RELEASE_ASSERT(funcType.args()[i].refType().isExtern()); + ScratchTagScope tag(masm, scratchV); + masm.splitTagForTest(scratchV, tag); + + // For object inputs, we handle object and null inline, everything + // else requires an actual box and we go out of line to allocate + // that. + masm.branchTestObject(Assembler::Equal, tag, &next); + masm.branchTestNull(Assembler::Equal, tag, &next); + masm.jump(&oolCall); + break; + } + case ValType::V128: { + // Guarded against by hasUnexposableArgOrRet() + MOZ_CRASH("unexpected argument type when calling from the jit"); + } + default: { + MOZ_CRASH("unexpected argument type when calling from the jit"); + } + } + masm.nopAlign(CodeAlignment); + masm.bind(&next); + } + + Label rejoinBeforeCall; + masm.bind(&rejoinBeforeCall); + + // Convert all the expected values to unboxed values on the stack. + ArgTypeVector args(funcType); + for (WasmABIArgIter iter(args); !iter.done(); iter++) { + Address argv(FramePointer, JitFrameLayout::offsetOfActualArg(iter.index())); + bool isStackArg = iter->kind() == ABIArg::Stack; + switch (iter.mirType()) { + case MIRType::Int32: { + Register target = isStackArg ? ScratchIonEntry : iter->gpr(); + masm.unboxInt32(argv, target); + GenPrintIsize(DebugChannel::Function, masm, target); + if (isStackArg) { + masm.storePtr(target, Address(sp, iter->offsetFromArgBase())); + } + break; + } + case MIRType::Int64: { + // The coercion has provided a BigInt value by this point, which + // we need to convert to an I64 here. + if (isStackArg) { + Address dst(sp, iter->offsetFromArgBase()); + Register src = scratchV.payloadOrValueReg(); +#if JS_BITS_PER_WORD == 64 + Register64 scratch64(scratchG); +#else + Register64 scratch64(scratchG, ABINonArgReg3); +#endif + masm.unboxBigInt(argv, src); + masm.loadBigInt64(src, scratch64); + GenPrintI64(DebugChannel::Function, masm, scratch64); + masm.store64(scratch64, dst); + } else { + Register src = scratchG; + Register64 target = iter->gpr64(); + masm.unboxBigInt(argv, src); + masm.loadBigInt64(src, target); + GenPrintI64(DebugChannel::Function, masm, target); + } + break; + } + case MIRType::Float32: { + FloatRegister target = isStackArg ? ABINonArgDoubleReg : iter->fpu(); + masm.unboxDouble(argv, ABINonArgDoubleReg); + masm.convertDoubleToFloat32(ABINonArgDoubleReg, target); + GenPrintF32(DebugChannel::Function, masm, target.asSingle()); + if (isStackArg) { + masm.storeFloat32(target, Address(sp, iter->offsetFromArgBase())); + } + break; + } + case MIRType::Double: { + FloatRegister target = isStackArg ? ABINonArgDoubleReg : iter->fpu(); + masm.unboxDouble(argv, target); + GenPrintF64(DebugChannel::Function, masm, target); + if (isStackArg) { + masm.storeDouble(target, Address(sp, iter->offsetFromArgBase())); + } + break; + } + case MIRType::RefOrNull: { + Register target = isStackArg ? ScratchIonEntry : iter->gpr(); + masm.unboxObjectOrNull(argv, target); + GenPrintPtr(DebugChannel::Function, masm, target); + if (isStackArg) { + masm.storePtr(target, Address(sp, iter->offsetFromArgBase())); + } + break; + } + default: { + MOZ_CRASH("unexpected input argument when calling from jit"); + } + } + } + + GenPrintf(DebugChannel::Function, masm, "\n"); + + // Setup wasm register state. + masm.loadWasmPinnedRegsFromInstance(); + + masm.storePtr(InstanceReg, Address(masm.getStackPointer(), + WasmCalleeInstanceOffsetBeforeCall)); + + // Call into the real function. Note that, due to the throw stub, instance + // and pinned registers may be clobbered. + masm.assertStackAlignment(WasmStackAlignment); + CallFuncExport(masm, fe, funcPtr); + masm.assertStackAlignment(WasmStackAlignment); + + // If InstanceReg is equal to the FailInstanceReg magic value (set by the + // throw stub), then report the exception to the JIT caller by jumping into + // the exception stub. + Label exception; + masm.branchPtr(Assembler::Equal, InstanceReg, Imm32(FailInstanceReg), + &exception); + + // Pop arguments. + masm.freeStack(frameSizeExclFP); + + GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; returns ", + fe.funcIndex()); + + // Store the return value in the JSReturnOperand. + const ValTypeVector& results = funcType.results(); + if (results.length() == 0) { + GenPrintf(DebugChannel::Function, masm, "void"); + masm.moveValue(UndefinedValue(), JSReturnOperand); + } else { + MOZ_ASSERT(results.length() == 1, "multi-value return to JS unimplemented"); + switch (results[0].kind()) { + case ValType::I32: + GenPrintIsize(DebugChannel::Function, masm, ReturnReg); + // No widening is required, as the value is boxed. + masm.boxNonDouble(JSVAL_TYPE_INT32, ReturnReg, JSReturnOperand); + break; + case ValType::F32: { + masm.canonicalizeFloat(ReturnFloat32Reg); + masm.convertFloat32ToDouble(ReturnFloat32Reg, ReturnDoubleReg); + GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg); + ScratchDoubleScope fpscratch(masm); + masm.boxDouble(ReturnDoubleReg, JSReturnOperand, fpscratch); + break; + } + case ValType::F64: { + masm.canonicalizeDouble(ReturnDoubleReg); + GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg); + ScratchDoubleScope fpscratch(masm); + masm.boxDouble(ReturnDoubleReg, JSReturnOperand, fpscratch); + break; + } + case ValType::I64: { + Label fail, done; + GenPrintI64(DebugChannel::Function, masm, ReturnReg64); + GenerateBigIntInitialization(masm, 0, ReturnReg64, scratchG, fe, &fail); + masm.boxNonDouble(JSVAL_TYPE_BIGINT, scratchG, JSReturnOperand); + masm.jump(&done); + masm.bind(&fail); + // Fixup the stack for the exception tail so that we can share it. + masm.reserveStack(frameSizeExclFP); + masm.jump(&exception); + masm.bind(&done); + // Un-fixup the stack for the benefit of the assertion below. + masm.setFramePushed(0); + break; + } + case ValType::V128: { + MOZ_CRASH("unexpected return type when calling from ion to wasm"); + } + case ValType::Ref: { + STATIC_ASSERT_ANYREF_IS_JSOBJECT; + // Per comment above, the call may have clobbered the instance + // register, so reload since unboxing will need it. + GenerateJitEntryLoadInstance(masm); + UnboxAnyrefIntoValueReg(masm, InstanceReg, ReturnReg, JSReturnOperand, + WasmJitEntryReturnScratch); + break; + } + } + } + + GenPrintf(DebugChannel::Function, masm, "\n"); + + MOZ_ASSERT(masm.framePushed() == 0); + + AssertExpectedSP(masm); + GenerateJitEntryEpilogue(masm, offsets); + MOZ_ASSERT(masm.framePushed() == 0); + + // Generate an OOL call to the C++ conversion path. + if (funcType.args().length()) { + masm.bind(&oolCall); + masm.setFramePushed(frameSize); + + // Baseline and Ion call C++ runtime via BuiltinThunk with wasm abi, so to + // unify the BuiltinThunk's interface we call it here with wasm abi. + jit::WasmABIArgIter argsIter(coerceArgTypes); + + // argument 0: function export index. + if (argsIter->kind() == ABIArg::GPR) { + masm.movePtr(ImmWord(funcExportIndex), argsIter->gpr()); + } else { + masm.storePtr(ImmWord(funcExportIndex), + Address(sp, argsIter->offsetFromArgBase())); + } + argsIter++; + + // argument 1: instance + if (argsIter->kind() == ABIArg::GPR) { + masm.movePtr(InstanceReg, argsIter->gpr()); + } else { + masm.storePtr(InstanceReg, Address(sp, argsIter->offsetFromArgBase())); + } + argsIter++; + + // argument 2: effective address of start of argv + Address argv(FramePointer, JitFrameLayout::offsetOfActualArgs()); + if (argsIter->kind() == ABIArg::GPR) { + masm.computeEffectiveAddress(argv, argsIter->gpr()); + } else { + masm.computeEffectiveAddress(argv, ScratchIonEntry); + masm.storePtr(ScratchIonEntry, + Address(sp, argsIter->offsetFromArgBase())); + } + argsIter++; + MOZ_ASSERT(argsIter.done()); + + masm.assertStackAlignment(ABIStackAlignment); + CallSymbolicAddress(masm, !fe.hasEagerStubs(), + SymbolicAddress::CoerceInPlace_JitEntry); + masm.assertStackAlignment(ABIStackAlignment); + + // No widening is required, as the return value is used as a bool. + masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, + &rejoinBeforeCall); + } + + // Prepare to throw: reload InstanceReg from the frame. + masm.bind(&exception); + masm.setFramePushed(frameSize); + GenerateJitEntryThrow(masm, frameSize); + + return FinishOffsets(masm, offsets); +} + +void wasm::GenerateDirectCallFromJit(MacroAssembler& masm, const FuncExport& fe, + const Instance& inst, + const JitCallStackArgVector& stackArgs, + Register scratch, uint32_t* callOffset) { + MOZ_ASSERT(!IsCompilingWasm()); + + const FuncType& funcType = inst.metadata().getFuncExportType(fe); + + size_t framePushedAtStart = masm.framePushed(); + + // Note, if code here pushes a reference value into the frame for its own + // purposes (and not just as an argument to the callee) then the frame must be + // traced in TraceJitExitFrame, see the case there for DirectWasmJitCall. The + // callee will trace values that are pushed as arguments, however. + + // Push a special frame descriptor that indicates the frame size so we can + // directly iterate from the current JIT frame without an extra call. + // Note: buildFakeExitFrame pushes an ExitFrameLayout containing the current + // frame pointer. We also use this to restore the frame pointer after the + // call. + *callOffset = masm.buildFakeExitFrame(scratch); + // FP := ExitFrameLayout* + masm.moveStackPtrTo(FramePointer); + masm.loadJSContext(scratch); + masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::DirectWasmJitCall); + + // Move stack arguments to their final locations. + unsigned bytesNeeded = StackArgBytesForWasmABI(funcType); + bytesNeeded = StackDecrementForCall(WasmStackAlignment, masm.framePushed(), + bytesNeeded); + if (bytesNeeded) { + masm.reserveStack(bytesNeeded); + } + + GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; arguments ", + fe.funcIndex()); + + ArgTypeVector args(funcType); + for (WasmABIArgIter iter(args); !iter.done(); iter++) { + MOZ_ASSERT_IF(iter->kind() == ABIArg::GPR, iter->gpr() != scratch); + MOZ_ASSERT_IF(iter->kind() == ABIArg::GPR, iter->gpr() != FramePointer); + if (iter->kind() != ABIArg::Stack) { + switch (iter.mirType()) { + case MIRType::Int32: + GenPrintIsize(DebugChannel::Function, masm, iter->gpr()); + break; + case MIRType::Int64: + GenPrintI64(DebugChannel::Function, masm, iter->gpr64()); + break; + case MIRType::Float32: + GenPrintF32(DebugChannel::Function, masm, iter->fpu()); + break; + case MIRType::Double: + GenPrintF64(DebugChannel::Function, masm, iter->fpu()); + break; + case MIRType::RefOrNull: + GenPrintPtr(DebugChannel::Function, masm, iter->gpr()); + break; + case MIRType::StackResults: + MOZ_ASSERT(args.isSyntheticStackResultPointerArg(iter.index())); + GenPrintPtr(DebugChannel::Function, masm, iter->gpr()); + break; + default: + MOZ_CRASH("ion to wasm fast path can only handle i32/f32/f64"); + } + continue; + } + + Address dst(masm.getStackPointer(), iter->offsetFromArgBase()); + + const JitCallStackArg& stackArg = stackArgs[iter.index()]; + switch (stackArg.tag()) { + case JitCallStackArg::Tag::Imm32: + GenPrintf(DebugChannel::Function, masm, "%d ", stackArg.imm32()); + masm.storePtr(ImmWord(stackArg.imm32()), dst); + break; + case JitCallStackArg::Tag::GPR: + MOZ_ASSERT(stackArg.gpr() != scratch); + MOZ_ASSERT(stackArg.gpr() != FramePointer); + GenPrintIsize(DebugChannel::Function, masm, stackArg.gpr()); + masm.storePtr(stackArg.gpr(), dst); + break; + case JitCallStackArg::Tag::FPU: + switch (iter.mirType()) { + case MIRType::Double: + GenPrintF64(DebugChannel::Function, masm, stackArg.fpu()); + masm.storeDouble(stackArg.fpu(), dst); + break; + case MIRType::Float32: + GenPrintF32(DebugChannel::Function, masm, stackArg.fpu()); + masm.storeFloat32(stackArg.fpu(), dst); + break; + default: + MOZ_CRASH( + "unexpected MIR type for a float register in wasm fast call"); + } + break; + case JitCallStackArg::Tag::Address: { + // The address offsets were valid *before* we pushed our frame. + Address src = stackArg.addr(); + MOZ_ASSERT(src.base == masm.getStackPointer()); + src.offset += masm.framePushed() - framePushedAtStart; + switch (iter.mirType()) { + case MIRType::Double: { + ScratchDoubleScope fpscratch(masm); + GenPrintF64(DebugChannel::Function, masm, fpscratch); + masm.loadDouble(src, fpscratch); + masm.storeDouble(fpscratch, dst); + break; + } + case MIRType::Float32: { + ScratchFloat32Scope fpscratch(masm); + masm.loadFloat32(src, fpscratch); + GenPrintF32(DebugChannel::Function, masm, fpscratch); + masm.storeFloat32(fpscratch, dst); + break; + } + case MIRType::Int32: { + masm.loadPtr(src, scratch); + GenPrintIsize(DebugChannel::Function, masm, scratch); + masm.storePtr(scratch, dst); + break; + } + case MIRType::RefOrNull: { + masm.loadPtr(src, scratch); + GenPrintPtr(DebugChannel::Function, masm, scratch); + masm.storePtr(scratch, dst); + break; + } + case MIRType::StackResults: { + MOZ_CRASH("multi-value in ion to wasm fast path unimplemented"); + } + default: { + MOZ_CRASH("unexpected MIR type for a stack slot in wasm fast call"); + } + } + break; + } + case JitCallStackArg::Tag::Undefined: { + MOZ_CRASH("can't happen because of arg.kind() check"); + } + } + } + + GenPrintf(DebugChannel::Function, masm, "\n"); + + // Load instance; from now on, InstanceReg is live. + masm.movePtr(ImmPtr(&inst), InstanceReg); + masm.storePtr(InstanceReg, Address(masm.getStackPointer(), + WasmCalleeInstanceOffsetBeforeCall)); + masm.loadWasmPinnedRegsFromInstance(); + + // Actual call. + const CodeTier& codeTier = inst.code().codeTier(inst.code().bestTier()); + const MetadataTier& metadata = codeTier.metadata(); + const CodeRange& codeRange = metadata.codeRange(fe); + void* callee = codeTier.segment().base() + codeRange.funcUncheckedCallEntry(); + + masm.assertStackAlignment(WasmStackAlignment); + MoveSPForJitABI(masm); + masm.callJit(ImmPtr(callee)); +#ifdef JS_CODEGEN_ARM64 + // WASM does not always keep PSP in sync with SP. So reinitialize it as it + // might be clobbered either by WASM or by any C++ calls within. + masm.initPseudoStackPtr(); +#endif + masm.assertStackAlignment(WasmStackAlignment); + + masm.branchPtr(Assembler::Equal, InstanceReg, Imm32(wasm::FailInstanceReg), + masm.exceptionLabel()); + + // Store the return value in the appropriate place. + GenPrintf(DebugChannel::Function, masm, "wasm-function[%d]; returns ", + fe.funcIndex()); + const ValTypeVector& results = funcType.results(); + if (results.length() == 0) { + masm.moveValue(UndefinedValue(), JSReturnOperand); + GenPrintf(DebugChannel::Function, masm, "void"); + } else { + MOZ_ASSERT(results.length() == 1, "multi-value return to JS unimplemented"); + switch (results[0].kind()) { + case wasm::ValType::I32: + // The return value is in ReturnReg, which is what Ion expects. + GenPrintIsize(DebugChannel::Function, masm, ReturnReg); +#ifdef JS_64BIT + masm.widenInt32(ReturnReg); +#endif + break; + case wasm::ValType::I64: + // The return value is in ReturnReg64, which is what Ion expects. + GenPrintI64(DebugChannel::Function, masm, ReturnReg64); + break; + case wasm::ValType::F32: + masm.canonicalizeFloat(ReturnFloat32Reg); + GenPrintF32(DebugChannel::Function, masm, ReturnFloat32Reg); + break; + case wasm::ValType::F64: + masm.canonicalizeDouble(ReturnDoubleReg); + GenPrintF64(DebugChannel::Function, masm, ReturnDoubleReg); + break; + case wasm::ValType::Ref: + STATIC_ASSERT_ANYREF_IS_JSOBJECT; + // The call to wasm above preserves the InstanceReg, we don't + // need to reload it here. + UnboxAnyrefIntoValueReg(masm, InstanceReg, ReturnReg, JSReturnOperand, + WasmJitEntryReturnScratch); + break; + case wasm::ValType::V128: + MOZ_CRASH("unexpected return type when calling from ion to wasm"); + } + } + + GenPrintf(DebugChannel::Function, masm, "\n"); + + // Restore the frame pointer. + masm.loadPtr(Address(FramePointer, 0), FramePointer); + + // Free args + frame descriptor. + masm.leaveExitFrame(bytesNeeded + ExitFrameLayout::Size()); + + MOZ_ASSERT(framePushedAtStart == masm.framePushed()); +} + +static void StackCopy(MacroAssembler& masm, MIRType type, Register scratch, + Address src, Address dst) { + if (type == MIRType::Int32) { + masm.load32(src, scratch); + GenPrintIsize(DebugChannel::Import, masm, scratch); + masm.store32(scratch, dst); + } else if (type == MIRType::Int64) { +#if JS_BITS_PER_WORD == 32 + MOZ_RELEASE_ASSERT(src.base != scratch && dst.base != scratch); + GenPrintf(DebugChannel::Import, masm, "i64("); + masm.load32(LowWord(src), scratch); + GenPrintIsize(DebugChannel::Import, masm, scratch); + masm.store32(scratch, LowWord(dst)); + masm.load32(HighWord(src), scratch); + GenPrintIsize(DebugChannel::Import, masm, scratch); + masm.store32(scratch, HighWord(dst)); + GenPrintf(DebugChannel::Import, masm, ") "); +#else + Register64 scratch64(scratch); + masm.load64(src, scratch64); + GenPrintIsize(DebugChannel::Import, masm, scratch); + masm.store64(scratch64, dst); +#endif + } else if (type == MIRType::RefOrNull || type == MIRType::Pointer || + type == MIRType::StackResults) { + masm.loadPtr(src, scratch); + GenPrintPtr(DebugChannel::Import, masm, scratch); + masm.storePtr(scratch, dst); + } else if (type == MIRType::Float32) { + ScratchFloat32Scope fpscratch(masm); + masm.loadFloat32(src, fpscratch); + GenPrintF32(DebugChannel::Import, masm, fpscratch); + masm.storeFloat32(fpscratch, dst); + } else if (type == MIRType::Double) { + ScratchDoubleScope fpscratch(masm); + masm.loadDouble(src, fpscratch); + GenPrintF64(DebugChannel::Import, masm, fpscratch); + masm.storeDouble(fpscratch, dst); +#ifdef ENABLE_WASM_SIMD + } else if (type == MIRType::Simd128) { + ScratchSimd128Scope fpscratch(masm); + masm.loadUnalignedSimd128(src, fpscratch); + GenPrintV128(DebugChannel::Import, masm, fpscratch); + masm.storeUnalignedSimd128(fpscratch, dst); +#endif + } else { + MOZ_CRASH("StackCopy: unexpected type"); + } +} + +static void FillArgumentArrayForInterpExit(MacroAssembler& masm, + unsigned funcImportIndex, + const FuncType& funcType, + unsigned argOffset, + Register scratch) { + // This is FrameWithInstances::sizeOf() - ShadowStackSpace because the latter + // is accounted for by the ABIArgIter. + const unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithInstances); + + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; arguments ", + funcImportIndex); + + ArgTypeVector args(funcType); + for (ABIArgIter i(args); !i.done(); i++) { + Address dst(masm.getStackPointer(), argOffset + i.index() * sizeof(Value)); + + MIRType type = i.mirType(); + MOZ_ASSERT(args.isSyntheticStackResultPointerArg(i.index()) == + (type == MIRType::StackResults)); + switch (i->kind()) { + case ABIArg::GPR: + if (type == MIRType::Int32) { + GenPrintIsize(DebugChannel::Import, masm, i->gpr()); + masm.store32(i->gpr(), dst); + } else if (type == MIRType::Int64) { + GenPrintI64(DebugChannel::Import, masm, i->gpr64()); + masm.store64(i->gpr64(), dst); + } else if (type == MIRType::RefOrNull) { + GenPrintPtr(DebugChannel::Import, masm, i->gpr()); + masm.storePtr(i->gpr(), dst); + } else if (type == MIRType::StackResults) { + GenPrintPtr(DebugChannel::Import, masm, i->gpr()); + masm.storePtr(i->gpr(), dst); + } else { + MOZ_CRASH( + "FillArgumentArrayForInterpExit, ABIArg::GPR: unexpected type"); + } + break; +#ifdef JS_CODEGEN_REGISTER_PAIR + case ABIArg::GPR_PAIR: + if (type == MIRType::Int64) { + GenPrintI64(DebugChannel::Import, masm, i->gpr64()); + masm.store64(i->gpr64(), dst); + } else { + MOZ_CRASH("wasm uses hardfp for function calls."); + } + break; +#endif + case ABIArg::FPU: { + FloatRegister srcReg = i->fpu(); + if (type == MIRType::Double) { + GenPrintF64(DebugChannel::Import, masm, srcReg); + masm.storeDouble(srcReg, dst); + } else if (type == MIRType::Float32) { + // Preserve the NaN pattern in the input. + GenPrintF32(DebugChannel::Import, masm, srcReg); + masm.storeFloat32(srcReg, dst); + } else if (type == MIRType::Simd128) { + // The value should never escape; the call will be stopped later as + // the import is being called. But we should generate something sane + // here for the boxed case since a debugger or the stack walker may + // observe something. + ScratchDoubleScope dscratch(masm); + masm.loadConstantDouble(0, dscratch); + GenPrintF64(DebugChannel::Import, masm, dscratch); + masm.storeDouble(dscratch, dst); + } else { + MOZ_CRASH("Unknown MIRType in wasm exit stub"); + } + break; + } + case ABIArg::Stack: { + Address src(FramePointer, + offsetFromFPToCallerStackArgs + i->offsetFromArgBase()); + if (type == MIRType::Simd128) { + // As above. StackCopy does not know this trick. + ScratchDoubleScope dscratch(masm); + masm.loadConstantDouble(0, dscratch); + GenPrintF64(DebugChannel::Import, masm, dscratch); + masm.storeDouble(dscratch, dst); + } else { + StackCopy(masm, type, scratch, src, dst); + } + break; + } + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + } + GenPrintf(DebugChannel::Import, masm, "\n"); +} + +// Note, this may destroy the values in incoming argument registers as a result +// of Spectre mitigation. +static void FillArgumentArrayForJitExit(MacroAssembler& masm, Register instance, + unsigned funcImportIndex, + const FuncType& funcType, + unsigned argOffset, Register scratch, + Register scratch2, Label* throwLabel) { + MOZ_ASSERT(scratch != scratch2); + + // This is FrameWithInstances::sizeOf() - ShadowStackSpace because the latter + // is accounted for by the ABIArgIter. + const unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithInstances); + + // This loop does not root the values that are being constructed in + // for the arguments. Allocations that are generated by code either + // in the loop or called from it should be NoGC allocations. + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; arguments ", + funcImportIndex); + + ArgTypeVector args(funcType); + for (ABIArgIter i(args); !i.done(); i++) { + Address dst(masm.getStackPointer(), argOffset + i.index() * sizeof(Value)); + + MIRType type = i.mirType(); + MOZ_ASSERT(args.isSyntheticStackResultPointerArg(i.index()) == + (type == MIRType::StackResults)); + switch (i->kind()) { + case ABIArg::GPR: + if (type == MIRType::Int32) { + GenPrintIsize(DebugChannel::Import, masm, i->gpr()); + masm.storeValue(JSVAL_TYPE_INT32, i->gpr(), dst); + } else if (type == MIRType::Int64) { + // FuncType::canHaveJitExit should prevent this. Also see comments + // at GenerateBigIntInitialization. + MOZ_CRASH("Should not happen"); + } else if (type == MIRType::RefOrNull) { + // This works also for FuncRef because it is distinguishable from + // a boxed AnyRef. + masm.movePtr(i->gpr(), scratch2); + UnboxAnyrefIntoValue(masm, instance, scratch2, dst, scratch); + } else if (type == MIRType::StackResults) { + MOZ_CRASH("Multi-result exit to JIT unimplemented"); + } else { + MOZ_CRASH( + "FillArgumentArrayForJitExit, ABIArg::GPR: unexpected type"); + } + break; +#ifdef JS_CODEGEN_REGISTER_PAIR + case ABIArg::GPR_PAIR: + if (type == MIRType::Int64) { + // FuncType::canHaveJitExit should prevent this. Also see comments + // at GenerateBigIntInitialization. + MOZ_CRASH("Should not happen"); + } else { + MOZ_CRASH("wasm uses hardfp for function calls."); + } + break; +#endif + case ABIArg::FPU: { + FloatRegister srcReg = i->fpu(); + if (type == MIRType::Double) { + // Preserve the NaN pattern in the input. + ScratchDoubleScope fpscratch(masm); + masm.moveDouble(srcReg, fpscratch); + masm.canonicalizeDouble(fpscratch); + GenPrintF64(DebugChannel::Import, masm, fpscratch); + masm.boxDouble(fpscratch, dst); + } else if (type == MIRType::Float32) { + // JS::Values can't store Float32, so convert to a Double. + ScratchDoubleScope fpscratch(masm); + masm.convertFloat32ToDouble(srcReg, fpscratch); + masm.canonicalizeDouble(fpscratch); + GenPrintF64(DebugChannel::Import, masm, fpscratch); + masm.boxDouble(fpscratch, dst); + } else if (type == MIRType::Simd128) { + // The value should never escape; the call will be stopped later as + // the import is being called. But we should generate something sane + // here for the boxed case since a debugger or the stack walker may + // observe something. + ScratchDoubleScope dscratch(masm); + masm.loadConstantDouble(0, dscratch); + GenPrintF64(DebugChannel::Import, masm, dscratch); + masm.boxDouble(dscratch, dst); + } else { + MOZ_CRASH("Unknown MIRType in wasm exit stub"); + } + break; + } + case ABIArg::Stack: { + Address src(FramePointer, + offsetFromFPToCallerStackArgs + i->offsetFromArgBase()); + if (type == MIRType::Int32) { + masm.load32(src, scratch); + GenPrintIsize(DebugChannel::Import, masm, scratch); + masm.storeValue(JSVAL_TYPE_INT32, scratch, dst); + } else if (type == MIRType::Int64) { + // FuncType::canHaveJitExit should prevent this. Also see comments + // at GenerateBigIntInitialization. + MOZ_CRASH("Should not happen"); + } else if (type == MIRType::RefOrNull) { + // This works also for FuncRef because it is distinguishable from a + // boxed AnyRef. + masm.loadPtr(src, scratch); + UnboxAnyrefIntoValue(masm, instance, scratch, dst, scratch2); + } else if (IsFloatingPointType(type)) { + ScratchDoubleScope dscratch(masm); + FloatRegister fscratch = dscratch.asSingle(); + if (type == MIRType::Float32) { + masm.loadFloat32(src, fscratch); + masm.convertFloat32ToDouble(fscratch, dscratch); + } else { + masm.loadDouble(src, dscratch); + } + masm.canonicalizeDouble(dscratch); + GenPrintF64(DebugChannel::Import, masm, dscratch); + masm.boxDouble(dscratch, dst); + } else if (type == MIRType::Simd128) { + // The value should never escape; the call will be stopped later as + // the import is being called. But we should generate something + // sane here for the boxed case since a debugger or the stack walker + // may observe something. + ScratchDoubleScope dscratch(masm); + masm.loadConstantDouble(0, dscratch); + GenPrintF64(DebugChannel::Import, masm, dscratch); + masm.boxDouble(dscratch, dst); + } else { + MOZ_CRASH( + "FillArgumentArrayForJitExit, ABIArg::Stack: unexpected type"); + } + break; + } + case ABIArg::Uninitialized: + MOZ_CRASH("Uninitialized ABIArg kind"); + } + } + GenPrintf(DebugChannel::Import, masm, "\n"); +} + +// Generate a wrapper function with the standard intra-wasm call ABI which +// simply calls an import. This wrapper function allows any import to be treated +// like a normal wasm function for the purposes of exports and table calls. In +// particular, the wrapper function provides: +// - a table entry, so JS imports can be put into tables +// - normal entries, so that, if the import is re-exported, an entry stub can +// be generated and called without any special cases +static bool GenerateImportFunction(jit::MacroAssembler& masm, + const FuncImport& fi, + const FuncType& funcType, + CallIndirectId callIndirectId, + FuncOffsets* offsets) { + AutoCreatedBy acb(masm, "wasm::GenerateImportFunction"); + + AssertExpectedSP(masm); + + GenerateFunctionPrologue(masm, callIndirectId, Nothing(), offsets); + + MOZ_ASSERT(masm.framePushed() == 0); + const unsigned sizeOfInstanceSlot = sizeof(void*); + unsigned framePushed = StackDecrementForCall( + WasmStackAlignment, + sizeof(Frame), // pushed by prologue + StackArgBytesForWasmABI(funcType) + sizeOfInstanceSlot); + masm.wasmReserveStackChecked(framePushed, BytecodeOffset(0)); + MOZ_ASSERT(masm.framePushed() == framePushed); + + masm.storePtr(InstanceReg, Address(masm.getStackPointer(), + framePushed - sizeOfInstanceSlot)); + + // The argument register state is already setup by our caller. We just need + // to be sure not to clobber it before the call. + Register scratch = ABINonArgReg0; + + // Copy our frame's stack arguments to the callee frame's stack argument. + // + // Note offsetFromFPToCallerStackArgs is sizeof(Frame) because the + // WasmABIArgIter accounts for both the ShadowStackSpace and the instance + // fields of FrameWithInstances. + unsigned offsetFromFPToCallerStackArgs = sizeof(Frame); + ArgTypeVector args(funcType); + for (WasmABIArgIter i(args); !i.done(); i++) { + if (i->kind() != ABIArg::Stack) { + continue; + } + + Address src(FramePointer, + offsetFromFPToCallerStackArgs + i->offsetFromArgBase()); + Address dst(masm.getStackPointer(), i->offsetFromArgBase()); + GenPrintf(DebugChannel::Import, masm, + "calling exotic import function with arguments: "); + StackCopy(masm, i.mirType(), scratch, src, dst); + GenPrintf(DebugChannel::Import, masm, "\n"); + } + + // Call the import exit stub. + CallSiteDesc desc(CallSiteDesc::Import); + MoveSPForJitABI(masm); + masm.wasmCallImport(desc, CalleeDesc::import(fi.instanceOffset())); + + // Restore the instance register and pinned regs, per wasm function ABI. + masm.loadPtr( + Address(masm.getStackPointer(), framePushed - sizeOfInstanceSlot), + InstanceReg); + masm.loadWasmPinnedRegsFromInstance(); + + // Restore cx->realm. + masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1); + + GenerateFunctionEpilogue(masm, framePushed, offsets); + return FinishOffsets(masm, offsets); +} + +static const unsigned STUBS_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024; + +bool wasm::GenerateImportFunctions(const ModuleEnvironment& env, + const FuncImportVector& imports, + CompiledCode* code) { + LifoAlloc lifo(STUBS_LIFO_DEFAULT_CHUNK_SIZE); + TempAllocator alloc(&lifo); + WasmMacroAssembler masm(alloc, env); + + for (uint32_t funcIndex = 0; funcIndex < imports.length(); funcIndex++) { + const FuncImport& fi = imports[funcIndex]; + const FuncType& funcType = *env.funcs[funcIndex].type; + CallIndirectId callIndirectId = CallIndirectId::forFunc(env, funcIndex); + + FuncOffsets offsets; + if (!GenerateImportFunction(masm, fi, funcType, callIndirectId, &offsets)) { + return false; + } + if (!code->codeRanges.emplaceBack(funcIndex, /* bytecodeOffset = */ 0, + offsets)) { + return false; + } + } + + masm.finish(); + if (masm.oom()) { + return false; + } + + return code->swap(masm); +} + +// Generate a stub that is called via the internal ABI derived from the +// signature of the import and calls into an appropriate callImport C++ +// function, having boxed all the ABI arguments into a homogeneous Value array. +static bool GenerateImportInterpExit(MacroAssembler& masm, const FuncImport& fi, + const FuncType& funcType, + uint32_t funcImportIndex, + Label* throwLabel, + CallableOffsets* offsets) { + AutoCreatedBy acb(masm, "GenerateImportInterpExit"); + + AssertExpectedSP(masm); + masm.setFramePushed(0); + + // Argument types for Instance::callImport_*: + static const MIRType typeArray[] = {MIRType::Pointer, // Instance* + MIRType::Pointer, // funcImportIndex + MIRType::Int32, // argc + MIRType::Pointer}; // argv + MIRTypeVector invokeArgTypes; + MOZ_ALWAYS_TRUE(invokeArgTypes.append(typeArray, std::size(typeArray))); + + // At the point of the call, the stack layout is: + // + // | stack args | padding | argv[] | padding | retaddr | caller stack | ... + // ^ + // +-- sp + // + // The padding between stack args and argv ensures that argv is aligned on a + // Value boundary. The padding between argv and retaddr ensures that sp is + // aligned. The caller stack includes a ShadowStackArea and the instance + // fields before the args, see WasmFrame.h. + // + // The 'double' alignment is correct since the argv[] is a Value array. + unsigned argOffset = + AlignBytes(StackArgBytesForNativeABI(invokeArgTypes), sizeof(double)); + // The abiArgCount includes a stack result pointer argument if needed. + unsigned abiArgCount = ArgTypeVector(funcType).lengthWithStackResults(); + unsigned argBytes = std::max(1, abiArgCount) * sizeof(Value); + unsigned framePushed = + StackDecrementForCall(ABIStackAlignment, + sizeof(Frame), // pushed by prologue + argOffset + argBytes); + + GenerateExitPrologue(masm, framePushed, ExitReason::Fixed::ImportInterp, + offsets); + + // Fill the argument array. + Register scratch = ABINonArgReturnReg0; + FillArgumentArrayForInterpExit(masm, funcImportIndex, funcType, argOffset, + scratch); + + // Prepare the arguments for the call to Instance::callImport_*. + ABIArgMIRTypeIter i(invokeArgTypes); + + // argument 0: Instance* + if (i->kind() == ABIArg::GPR) { + masm.movePtr(InstanceReg, i->gpr()); + } else { + masm.storePtr(InstanceReg, + Address(masm.getStackPointer(), i->offsetFromArgBase())); + } + i++; + + // argument 1: funcImportIndex + if (i->kind() == ABIArg::GPR) { + masm.mov(ImmWord(funcImportIndex), i->gpr()); + } else { + masm.store32(Imm32(funcImportIndex), + Address(masm.getStackPointer(), i->offsetFromArgBase())); + } + i++; + + // argument 2: argc + unsigned argc = abiArgCount; + if (i->kind() == ABIArg::GPR) { + masm.mov(ImmWord(argc), i->gpr()); + } else { + masm.store32(Imm32(argc), + Address(masm.getStackPointer(), i->offsetFromArgBase())); + } + i++; + + // argument 3: argv + Address argv(masm.getStackPointer(), argOffset); + if (i->kind() == ABIArg::GPR) { + masm.computeEffectiveAddress(argv, i->gpr()); + } else { + masm.computeEffectiveAddress(argv, scratch); + masm.storePtr(scratch, + Address(masm.getStackPointer(), i->offsetFromArgBase())); + } + i++; + MOZ_ASSERT(i.done()); + + // Make the call, test whether it succeeded, and extract the return value. + AssertStackAlignment(masm, ABIStackAlignment); + masm.call(SymbolicAddress::CallImport_General); + masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); + + ResultType resultType = ResultType::Vector(funcType.results()); + ValType registerResultType; + for (ABIResultIter iter(resultType); !iter.done(); iter.next()) { + if (iter.cur().inRegister()) { + MOZ_ASSERT(!registerResultType.isValid()); + registerResultType = iter.cur().type(); + } + } + if (!registerResultType.isValid()) { + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + GenPrintf(DebugChannel::Import, masm, "void"); + } else { + switch (registerResultType.kind()) { + case ValType::I32: + masm.load32(argv, ReturnReg); + // No widening is required, as we know the value comes from an i32 load. + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + GenPrintIsize(DebugChannel::Import, masm, ReturnReg); + break; + case ValType::I64: + masm.load64(argv, ReturnReg64); + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + GenPrintI64(DebugChannel::Import, masm, ReturnReg64); + break; + case ValType::V128: + // Note, CallImport_Rtt/V128 currently always throws, so we should never + // reach this point. + masm.breakpoint(); + break; + case ValType::F32: + masm.loadFloat32(argv, ReturnFloat32Reg); + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + GenPrintF32(DebugChannel::Import, masm, ReturnFloat32Reg); + break; + case ValType::F64: + masm.loadDouble(argv, ReturnDoubleReg); + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + GenPrintF64(DebugChannel::Import, masm, ReturnDoubleReg); + break; + case ValType::Ref: + STATIC_ASSERT_ANYREF_IS_JSOBJECT; + masm.loadPtr(argv, ReturnReg); + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + GenPrintPtr(DebugChannel::Import, masm, ReturnReg); + break; + } + } + + GenPrintf(DebugChannel::Import, masm, "\n"); + + // The native ABI preserves the instance, heap and global registers since they + // are non-volatile. + MOZ_ASSERT(NonVolatileRegs.has(InstanceReg)); +#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \ + defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) + MOZ_ASSERT(NonVolatileRegs.has(HeapReg)); +#endif + + GenerateExitEpilogue(masm, framePushed, ExitReason::Fixed::ImportInterp, + offsets); + + return FinishOffsets(masm, offsets); +} + +// Generate a stub that is called via the internal ABI derived from the +// signature of the import and calls into a compatible JIT function, +// having boxed all the ABI arguments into the JIT stack frame layout. +static bool GenerateImportJitExit(MacroAssembler& masm, const FuncImport& fi, + const FuncType& funcType, + unsigned funcImportIndex, Label* throwLabel, + CallableOffsets* offsets) { + AutoCreatedBy acb(masm, "GenerateImportJitExit"); + + AssertExpectedSP(masm); + masm.setFramePushed(0); + + // JIT calls use the following stack layout: + // + // | WasmToJSJitFrameLayout | this | arg1..N | saved instance | ... + // ^ + // +-- sp + // + // The JIT ABI requires that sp be JitStackAlignment-aligned after pushing + // the return address and frame pointer. + static_assert(WasmStackAlignment >= JitStackAlignment, "subsumes"); + const unsigned sizeOfInstanceSlot = sizeof(void*); + const unsigned sizeOfRetAddrAndFP = 2 * sizeof(void*); + const unsigned sizeOfPreFrame = + WasmToJSJitFrameLayout::Size() - sizeOfRetAddrAndFP; + const unsigned sizeOfThisAndArgs = + (1 + funcType.args().length()) * sizeof(Value); + const unsigned totalJitFrameBytes = sizeOfRetAddrAndFP + sizeOfPreFrame + + sizeOfThisAndArgs + sizeOfInstanceSlot; + const unsigned jitFramePushed = + StackDecrementForCall(JitStackAlignment, + sizeof(Frame), // pushed by prologue + totalJitFrameBytes) - + sizeOfRetAddrAndFP; + + GenerateJitExitPrologue(masm, jitFramePushed, offsets); + + // 1. Descriptor. + unsigned argc = funcType.args().length(); + size_t argOffset = 0; + uint32_t descriptor = + MakeFrameDescriptorForJitCall(FrameType::WasmToJSJit, argc); + masm.storePtr(ImmWord(uintptr_t(descriptor)), + Address(masm.getStackPointer(), argOffset)); + argOffset += sizeof(size_t); + + // 2. Callee, part 1 -- need the callee register for argument filling, so + // record offset here and set up callee later. + size_t calleeArgOffset = argOffset; + argOffset += sizeof(size_t); + MOZ_ASSERT(argOffset == sizeOfPreFrame); + + // 3. |this| value. + masm.storeValue(UndefinedValue(), Address(masm.getStackPointer(), argOffset)); + argOffset += sizeof(Value); + + // 4. Fill the arguments. + Register scratch = ABINonArgReturnReg1; // Repeatedly clobbered + Register scratch2 = ABINonArgReturnReg0; // Reused as callee below + FillArgumentArrayForJitExit(masm, InstanceReg, funcImportIndex, funcType, + argOffset, scratch, scratch2, throwLabel); + argOffset += funcType.args().length() * sizeof(Value); + MOZ_ASSERT(argOffset == sizeOfThisAndArgs + sizeOfPreFrame); + + // Preserve instance because the JIT callee clobbers it. + const size_t savedInstanceOffset = argOffset; + masm.storePtr(InstanceReg, + Address(masm.getStackPointer(), savedInstanceOffset)); + + // 2. Callee, part 2 -- now that the register is free, set up the callee. + Register callee = ABINonArgReturnReg0; // Live until call + + // 2.1. Get the callee. This must be a JSFunction if we're using this JIT + // exit. + masm.loadPtr( + Address(InstanceReg, Instance::offsetInData( + fi.instanceOffset() + + offsetof(FuncImportInstanceData, callable))), + callee); + + // 2.2. Save callee. + masm.storePtr(callee, Address(masm.getStackPointer(), calleeArgOffset)); + + // 5. Check if we need to rectify arguments. + masm.loadFunctionArgCount(callee, scratch); + + Label rectify; + masm.branch32(Assembler::Above, scratch, Imm32(funcType.args().length()), + &rectify); + + // 6. If we haven't rectified arguments, load callee executable entry point. + + masm.loadJitCodeRaw(callee, callee); + + Label rejoinBeforeCall; + masm.bind(&rejoinBeforeCall); + + AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddrAndFP); +#ifdef JS_CODEGEN_ARM64 + AssertExpectedSP(masm); + // Manually resync PSP. Omitting this causes eg tests/wasm/import-export.js + // to segfault. + masm.moveStackPtrTo(PseudoStackPointer); +#endif + masm.callJitNoProfiler(callee); + + // Note that there might be a GC thing in the JSReturnOperand now. + // In all the code paths from here: + // - either the value is unboxed because it was a primitive and we don't + // need to worry about rooting anymore. + // - or the value needs to be rooted, but nothing can cause a GC between + // here and CoerceInPlace, which roots before coercing to a primitive. + + // The JIT callee clobbers all registers other than the frame pointer, so + // restore InstanceReg here. + AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddrAndFP); + masm.loadPtr(Address(masm.getStackPointer(), savedInstanceOffset), + InstanceReg); + + // The frame was aligned for the JIT ABI such that + // (sp - 2 * sizeof(void*)) % JitStackAlignment == 0 + // But now we possibly want to call one of several different C++ functions, + // so subtract 2 * sizeof(void*) so that sp is aligned for an ABI call. + static_assert(ABIStackAlignment <= JitStackAlignment, "subsumes"); + masm.reserveStack(sizeOfRetAddrAndFP); + unsigned nativeFramePushed = masm.framePushed(); + AssertStackAlignment(masm, ABIStackAlignment); + +#ifdef DEBUG + { + Label ok; + masm.branchTestMagic(Assembler::NotEqual, JSReturnOperand, &ok); + masm.breakpoint(); + masm.bind(&ok); + } +#endif + + GenPrintf(DebugChannel::Import, masm, "wasm-import[%u]; returns ", + funcImportIndex); + + Label oolConvert; + const ValTypeVector& results = funcType.results(); + if (results.length() == 0) { + GenPrintf(DebugChannel::Import, masm, "void"); + } else { + MOZ_ASSERT(results.length() == 1, "multi-value return unimplemented"); + switch (results[0].kind()) { + case ValType::I32: + // No widening is required, as the return value does not come to us in + // ReturnReg. + masm.truncateValueToInt32(JSReturnOperand, ReturnDoubleReg, ReturnReg, + &oolConvert); + GenPrintIsize(DebugChannel::Import, masm, ReturnReg); + break; + case ValType::I64: + // No fastpath for now, go immediately to ool case + masm.jump(&oolConvert); + break; + case ValType::V128: + // Unreachable as callImport should not call the stub. + masm.breakpoint(); + break; + case ValType::F32: + masm.convertValueToFloat(JSReturnOperand, ReturnFloat32Reg, + &oolConvert); + GenPrintF32(DebugChannel::Import, masm, ReturnFloat32Reg); + break; + case ValType::F64: + masm.convertValueToDouble(JSReturnOperand, ReturnDoubleReg, + &oolConvert); + GenPrintF64(DebugChannel::Import, masm, ReturnDoubleReg); + break; + case ValType::Ref: + // Guarded by temporarilyUnsupportedReftypeForExit() + MOZ_RELEASE_ASSERT(results[0].refType().isExtern()); + BoxValueIntoAnyref(masm, JSReturnOperand, ReturnReg, &oolConvert); + GenPrintPtr(DebugChannel::Import, masm, ReturnReg); + break; + } + } + + GenPrintf(DebugChannel::Import, masm, "\n"); + + Label done; + masm.bind(&done); + + GenerateJitExitEpilogue(masm, masm.framePushed(), offsets); + + { + // Call the arguments rectifier. + masm.bind(&rectify); + masm.loadPtr(Address(InstanceReg, Instance::offsetOfJSJitArgsRectifier()), + callee); + masm.jump(&rejoinBeforeCall); + } + + if (oolConvert.used()) { + masm.bind(&oolConvert); + masm.setFramePushed(nativeFramePushed); + + // Coercion calls use the following stack layout (sp grows to the left): + // | args | padding | Value argv[1] | padding | exit Frame | + MIRTypeVector coerceArgTypes; + MOZ_ALWAYS_TRUE(coerceArgTypes.append(MIRType::Pointer)); + unsigned offsetToCoerceArgv = + AlignBytes(StackArgBytesForNativeABI(coerceArgTypes), sizeof(Value)); + MOZ_ASSERT(nativeFramePushed >= offsetToCoerceArgv + sizeof(Value)); + AssertStackAlignment(masm, ABIStackAlignment); + + // Store return value into argv[0]. + masm.storeValue(JSReturnOperand, + Address(masm.getStackPointer(), offsetToCoerceArgv)); + + // From this point, it's safe to reuse the scratch register (which + // might be part of the JSReturnOperand). + + // The JIT might have clobbered exitFP at this point. Since there's + // going to be a CoerceInPlace call, pretend we're still doing the JIT + // call by restoring our tagged exitFP. + SetExitFP(masm, ExitReason::Fixed::ImportJit, scratch); + + // argument 0: argv + ABIArgMIRTypeIter i(coerceArgTypes); + Address argv(masm.getStackPointer(), offsetToCoerceArgv); + if (i->kind() == ABIArg::GPR) { + masm.computeEffectiveAddress(argv, i->gpr()); + } else { + masm.computeEffectiveAddress(argv, scratch); + masm.storePtr(scratch, + Address(masm.getStackPointer(), i->offsetFromArgBase())); + } + i++; + MOZ_ASSERT(i.done()); + + // Call coercion function. Note that right after the call, the value of + // FP is correct because FP is non-volatile in the native ABI. + AssertStackAlignment(masm, ABIStackAlignment); + const ValTypeVector& results = funcType.results(); + if (results.length() > 0) { + // NOTE that once there can be more than one result and we can box some of + // the results (as we must for AnyRef), pointer and already-boxed results + // must be rooted while subsequent results are boxed. + MOZ_ASSERT(results.length() == 1, "multi-value return unimplemented"); + switch (results[0].kind()) { + case ValType::I32: + masm.call(SymbolicAddress::CoerceInPlace_ToInt32); + masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); + masm.unboxInt32(Address(masm.getStackPointer(), offsetToCoerceArgv), + ReturnReg); + // No widening is required, as we generate a known-good value in a + // safe way here. + break; + case ValType::I64: { + masm.call(SymbolicAddress::CoerceInPlace_ToBigInt); + masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); + Address argv(masm.getStackPointer(), offsetToCoerceArgv); + masm.unboxBigInt(argv, scratch); + masm.loadBigInt64(scratch, ReturnReg64); + break; + } + case ValType::F64: + case ValType::F32: + masm.call(SymbolicAddress::CoerceInPlace_ToNumber); + masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); + masm.unboxDouble(Address(masm.getStackPointer(), offsetToCoerceArgv), + ReturnDoubleReg); + if (results[0].kind() == ValType::F32) { + masm.convertDoubleToFloat32(ReturnDoubleReg, ReturnFloat32Reg); + } + break; + case ValType::Ref: + // Guarded by temporarilyUnsupportedReftypeForExit() + MOZ_RELEASE_ASSERT(results[0].refType().isExtern()); + masm.call(SymbolicAddress::BoxValue_Anyref); + masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); + break; + default: + MOZ_CRASH("Unsupported convert type"); + } + } + + // Maintain the invariant that exitFP is either unset or not set to a + // wasm tagged exitFP, per the jit exit contract. + ClearExitFP(masm, scratch); + + masm.jump(&done); + masm.setFramePushed(0); + } + + MOZ_ASSERT(masm.framePushed() == 0); + + return FinishOffsets(masm, offsets); +} + +struct ABIFunctionArgs { + ABIFunctionType abiType; + size_t len; + + explicit ABIFunctionArgs(ABIFunctionType sig) + : abiType(ABIFunctionType(sig >> ArgType_Shift)) { + len = 0; + uint64_t i = uint64_t(abiType); + while (i) { + i = i >> ArgType_Shift; + len++; + } + } + + size_t length() const { return len; } + + MIRType operator[](size_t i) const { + MOZ_ASSERT(i < len); + uint64_t abi = uint64_t(abiType); + while (i--) { + abi = abi >> ArgType_Shift; + } + return ToMIRType(ABIArgType(abi & ArgType_Mask)); + } +}; + +bool wasm::GenerateBuiltinThunk(MacroAssembler& masm, ABIFunctionType abiType, + ExitReason exitReason, void* funcPtr, + CallableOffsets* offsets) { + AssertExpectedSP(masm); + masm.setFramePushed(0); + + ABIFunctionArgs args(abiType); + uint32_t framePushed = + StackDecrementForCall(ABIStackAlignment, + sizeof(Frame), // pushed by prologue + StackArgBytesForNativeABI(args)); + + GenerateExitPrologue(masm, framePushed, exitReason, offsets); + + // Copy out and convert caller arguments, if needed. + + // This is FrameWithInstances::sizeOf() - ShadowStackSpace because the latter + // is accounted for by the ABIArgIter. + unsigned offsetFromFPToCallerStackArgs = sizeof(FrameWithInstances); + Register scratch = ABINonArgReturnReg0; + for (ABIArgIter i(args); !i.done(); i++) { + if (i->argInRegister()) { +#ifdef JS_CODEGEN_ARM + // Non hard-fp passes the args values in GPRs. + if (!UseHardFpABI() && IsFloatingPointType(i.mirType())) { + FloatRegister input = i->fpu(); + if (i.mirType() == MIRType::Float32) { + masm.ma_vxfer(input, Register::FromCode(input.id())); + } else if (i.mirType() == MIRType::Double) { + uint32_t regId = input.singleOverlay().id(); + masm.ma_vxfer(input, Register::FromCode(regId), + Register::FromCode(regId + 1)); + } + } +#endif + continue; + } + + Address src(FramePointer, + offsetFromFPToCallerStackArgs + i->offsetFromArgBase()); + Address dst(masm.getStackPointer(), i->offsetFromArgBase()); + StackCopy(masm, i.mirType(), scratch, src, dst); + } + + AssertStackAlignment(masm, ABIStackAlignment); + MoveSPForJitABI(masm); + masm.call(ImmPtr(funcPtr, ImmPtr::NoCheckToken())); + +#if defined(JS_CODEGEN_X64) + // No widening is required, as the caller will widen. +#elif defined(JS_CODEGEN_X86) + // x86 passes the return value on the x87 FP stack. + Operand op(esp, 0); + MIRType retType = ToMIRType(ABIArgType( + std::underlying_type_t(abiType) & ArgType_Mask)); + if (retType == MIRType::Float32) { + masm.fstp32(op); + masm.loadFloat32(op, ReturnFloat32Reg); + } else if (retType == MIRType::Double) { + masm.fstp(op); + masm.loadDouble(op, ReturnDoubleReg); + } +#elif defined(JS_CODEGEN_ARM) + // Non hard-fp passes the return values in GPRs. + MIRType retType = ToMIRType(ABIArgType( + std::underlying_type_t(abiType) & ArgType_Mask)); + if (!UseHardFpABI() && IsFloatingPointType(retType)) { + masm.ma_vxfer(r0, r1, d0); + } +#endif + + GenerateExitEpilogue(masm, framePushed, exitReason, offsets); + return FinishOffsets(masm, offsets); +} + +#if defined(JS_CODEGEN_ARM) +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~((Registers::SetType(1) << Registers::sp) | + (Registers::SetType(1) << Registers::pc))), + FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# ifdef ENABLE_WASM_SIMD +# error "high lanes of SIMD registers need to be saved too." +# endif +#elif defined(JS_CODEGEN_MIPS64) +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~((Registers::SetType(1) << Registers::k0) | + (Registers::SetType(1) << Registers::k1) | + (Registers::SetType(1) << Registers::sp) | + (Registers::SetType(1) << Registers::zero))), + FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# ifdef ENABLE_WASM_SIMD +# error "high lanes of SIMD registers need to be saved too." +# endif +#elif defined(JS_CODEGEN_LOONG64) +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~((uint32_t(1) << Registers::tp) | + (uint32_t(1) << Registers::fp) | + (uint32_t(1) << Registers::sp) | + (uint32_t(1) << Registers::zero))), + FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# ifdef ENABLE_WASM_SIMD +# error "high lanes of SIMD registers need to be saved too." +# endif +#elif defined(JS_CODEGEN_RISCV64) +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~((uint32_t(1) << Registers::tp) | + (uint32_t(1) << Registers::fp) | + (uint32_t(1) << Registers::sp) | + (uint32_t(1) << Registers::zero))), + FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# ifdef ENABLE_WASM_SIMD +# error "high lanes of SIMD registers need to be saved too." +# endif +#elif defined(JS_CODEGEN_ARM64) +// We assume that traps do not happen while lr is live. This both ensures that +// the size of RegsToPreserve is a multiple of 2 (preserving WasmStackAlignment) +// and gives us a register to clobber in the return path. +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~((Registers::SetType(1) << RealStackPointer.code()) | + (Registers::SetType(1) << Registers::lr))), +# ifdef ENABLE_WASM_SIMD + FloatRegisterSet(FloatRegisters::AllSimd128Mask)); +# else + // If SIMD is not enabled, it's pointless to save/restore the upper 64 + // bits of each vector register. + FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# endif +#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) +// It's correct to use FloatRegisters::AllMask even when SIMD is not enabled; +// PushRegsInMask strips out the high lanes of the XMM registers in this case, +// while the singles will be stripped as they are aliased by the larger doubles. +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(Registers::AllMask & + ~(Registers::SetType(1) << Registers::StackPointer)), + FloatRegisterSet(FloatRegisters::AllMask)); +#else +static const LiveRegisterSet RegsToPreserve( + GeneralRegisterSet(0), FloatRegisterSet(FloatRegisters::AllDoubleMask)); +# ifdef ENABLE_WASM_SIMD +# error "no SIMD support" +# endif +#endif + +// Generate a RegisterOffsets which describes the locations of the GPRs as saved +// by GenerateTrapExit. FP registers are ignored. Note that the values +// stored in the RegisterOffsets are offsets in words downwards from the top of +// the save area. That is, a higher value implies a lower address. +void wasm::GenerateTrapExitRegisterOffsets(RegisterOffsets* offsets, + size_t* numWords) { + // This is the number of words pushed by the initial WasmPush(). + *numWords = WasmPushSize / sizeof(void*); + MOZ_ASSERT(*numWords == TrapExitDummyValueOffsetFromTop + 1); + + // And these correspond to the PushRegsInMask() that immediately follows. + for (GeneralRegisterBackwardIterator iter(RegsToPreserve.gprs()); iter.more(); + ++iter) { + offsets->setOffset(*iter, *numWords); + (*numWords)++; + } +} + +// Generate a stub which calls WasmReportTrap() and can be executed by having +// the signal handler redirect PC from any trapping instruction. +static bool GenerateTrapExit(MacroAssembler& masm, Label* throwLabel, + Offsets* offsets) { + AssertExpectedSP(masm); + masm.haltingAlign(CodeAlignment); + + masm.setFramePushed(0); + + offsets->begin = masm.currentOffset(); + + // Traps can only happen at well-defined program points. However, since + // traps may resume and the optimal assumption for the surrounding code is + // that registers are not clobbered, we need to preserve all registers in + // the trap exit. One simplifying assumption is that flags may be clobbered. + // Push a dummy word to use as return address below. + WasmPush(masm, ImmWord(TrapExitDummyValue)); + unsigned framePushedBeforePreserve = masm.framePushed(); + masm.PushRegsInMask(RegsToPreserve); + unsigned offsetOfReturnWord = masm.framePushed() - framePushedBeforePreserve; + + // We know that StackPointer is word-aligned, but not necessarily + // stack-aligned, so we need to align it dynamically. + Register preAlignStackPointer = ABINonVolatileReg; + masm.moveStackPtrTo(preAlignStackPointer); + masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1))); + if (ShadowStackSpace) { + masm.subFromStackPtr(Imm32(ShadowStackSpace)); + } + + masm.assertStackAlignment(ABIStackAlignment); + masm.call(SymbolicAddress::HandleTrap); + + // WasmHandleTrap returns null if control should transfer to the throw stub. + masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); + + // Otherwise, the return value is the TrapData::resumePC we must jump to. + // We must restore register state before jumping, which will clobber + // ReturnReg, so store ReturnReg in the above-reserved stack slot which we + // use to jump to via ret. + masm.moveToStackPtr(preAlignStackPointer); + masm.storePtr(ReturnReg, Address(masm.getStackPointer(), offsetOfReturnWord)); + masm.PopRegsInMask(RegsToPreserve); +#ifdef JS_CODEGEN_ARM64 + WasmPop(masm, lr); + masm.abiret(); +#else + masm.ret(); +#endif + + return FinishOffsets(masm, offsets); +} + +static void ClobberWasmRegsForLongJmp(MacroAssembler& masm, Register jumpReg) { + // Get the set of all registers that are allocatable in wasm functions + AllocatableGeneralRegisterSet gprs(GeneralRegisterSet::All()); + RegisterAllocator::takeWasmRegisters(gprs); + // Remove the instance register from this set as landing pads require it to be + // valid + gprs.take(InstanceReg); + // Remove a specified register that will be used for the longjmp + gprs.take(jumpReg); + // Set all of these registers to zero + for (GeneralRegisterIterator iter(gprs.asLiveSet()); iter.more(); ++iter) { + Register reg = *iter; + masm.xorPtr(reg, reg); + } + + // Get the set of all floating point registers that are allocatable in wasm + // functions + AllocatableFloatRegisterSet fprs(FloatRegisterSet::All()); + // Set all of these registers to NaN. We attempt for this to be a signalling + // NaN, but the bit format for signalling NaNs are implementation defined + // and so this is just best effort. + Maybe regNaN; + for (FloatRegisterIterator iter(fprs.asLiveSet()); iter.more(); ++iter) { + FloatRegister reg = *iter; + if (!reg.isDouble()) { + continue; + } + if (regNaN) { + masm.moveDouble(*regNaN, reg); + continue; + } + masm.loadConstantDouble(std::numeric_limits::signaling_NaN(), reg); + regNaN = Some(reg); + } +} + +// Generate a stub that restores the stack pointer to what it was on entry to +// the wasm activation, sets the return register to 'false' and then executes a +// return which will return from this wasm activation to the caller. This stub +// should only be called after the caller has reported an error. +static bool GenerateThrowStub(MacroAssembler& masm, Label* throwLabel, + Offsets* offsets) { + Register scratch1 = ABINonArgReturnReg0; + Register scratch2 = ABINonArgReturnReg1; + + AssertExpectedSP(masm); + masm.haltingAlign(CodeAlignment); + masm.setFramePushed(0); + + masm.bind(throwLabel); + + offsets->begin = masm.currentOffset(); + + // Conservatively, the stack pointer can be unaligned and we must align it + // dynamically. + masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1))); + if (ShadowStackSpace) { + masm.subFromStackPtr(Imm32(ShadowStackSpace)); + } + + // Allocate space for exception or regular resume information. + masm.reserveStack(sizeof(jit::ResumeFromException)); + masm.moveStackPtrTo(scratch1); + + MIRTypeVector handleThrowTypes; + MOZ_ALWAYS_TRUE(handleThrowTypes.append(MIRType::Pointer)); + + unsigned frameSize = + StackDecrementForCall(ABIStackAlignment, masm.framePushed(), + StackArgBytesForNativeABI(handleThrowTypes)); + masm.reserveStack(frameSize); + masm.assertStackAlignment(ABIStackAlignment); + + ABIArgMIRTypeIter i(handleThrowTypes); + if (i->kind() == ABIArg::GPR) { + masm.movePtr(scratch1, i->gpr()); + } else { + masm.storePtr(scratch1, + Address(masm.getStackPointer(), i->offsetFromArgBase())); + } + i++; + MOZ_ASSERT(i.done()); + + // WasmHandleThrow unwinds JitActivation::wasmExitFP() and returns the + // address of the return address on the stack this stub should return to. + // Set the FramePointer to a magic value to indicate a return by throw. + // + // If there is a Wasm catch handler present, it will instead return the + // address of the handler to jump to and the FP/SP values to restore. + masm.call(SymbolicAddress::HandleThrow); + + Label resumeCatch, leaveWasm; + + masm.load32(Address(ReturnReg, offsetof(jit::ResumeFromException, kind)), + scratch1); + + masm.branch32(Assembler::Equal, scratch1, + Imm32(jit::ExceptionResumeKind::WasmCatch), &resumeCatch); + masm.branch32(Assembler::Equal, scratch1, + Imm32(jit::ExceptionResumeKind::Wasm), &leaveWasm); + + masm.breakpoint(); + + // The case where a Wasm catch handler was found while unwinding the stack. + masm.bind(&resumeCatch); + masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfInstance()), + InstanceReg); + masm.loadWasmPinnedRegsFromInstance(); + masm.switchToWasmInstanceRealm(scratch1, scratch2); + masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfTarget()), + scratch1); + masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfFramePointer()), + FramePointer); + masm.loadStackPtr( + Address(ReturnReg, ResumeFromException::offsetOfStackPointer())); + MoveSPForJitABI(masm); + ClobberWasmRegsForLongJmp(masm, scratch1); + masm.jump(scratch1); + + // No catch handler was found, so we will just return out. + masm.bind(&leaveWasm); + masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfFramePointer()), + FramePointer); + masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfInstance()), + InstanceReg); + masm.loadPtr(Address(ReturnReg, ResumeFromException::offsetOfStackPointer()), + scratch1); + masm.moveToStackPtr(scratch1); +#ifdef JS_CODEGEN_ARM64 + masm.loadPtr(Address(scratch1, 0), lr); + masm.addToStackPtr(Imm32(8)); + masm.abiret(); +#else + masm.ret(); +#endif + + return FinishOffsets(masm, offsets); +} + +static const LiveRegisterSet AllAllocatableRegs = + LiveRegisterSet(GeneralRegisterSet(Registers::AllocatableMask), + FloatRegisterSet(FloatRegisters::AllMask)); + +// Generate a stub that handle toggable enter/leave frame traps or breakpoints. +// The trap records frame pointer (via GenerateExitPrologue) and saves most of +// registers to not affect the code generated by WasmBaselineCompile. +static bool GenerateDebugTrapStub(MacroAssembler& masm, Label* throwLabel, + CallableOffsets* offsets) { + AssertExpectedSP(masm); + masm.haltingAlign(CodeAlignment); + masm.setFramePushed(0); + + GenerateExitPrologue(masm, 0, ExitReason::Fixed::DebugTrap, offsets); + + // Save all registers used between baseline compiler operations. + masm.PushRegsInMask(AllAllocatableRegs); + + uint32_t framePushed = masm.framePushed(); + + // This method might be called with unaligned stack -- aligning and + // saving old stack pointer at the top. +#ifdef JS_CODEGEN_ARM64 + // On ARM64 however the stack is always aligned. + static_assert(ABIStackAlignment == 16, "ARM64 SP alignment"); +#else + Register scratch = ABINonArgReturnReg0; + masm.moveStackPtrTo(scratch); + masm.subFromStackPtr(Imm32(sizeof(intptr_t))); + masm.andToStackPtr(Imm32(~(ABIStackAlignment - 1))); + masm.storePtr(scratch, Address(masm.getStackPointer(), 0)); +#endif + + if (ShadowStackSpace) { + masm.subFromStackPtr(Imm32(ShadowStackSpace)); + } + masm.assertStackAlignment(ABIStackAlignment); + masm.call(SymbolicAddress::HandleDebugTrap); + + masm.branchIfFalseBool(ReturnReg, throwLabel); + + if (ShadowStackSpace) { + masm.addToStackPtr(Imm32(ShadowStackSpace)); + } +#ifndef JS_CODEGEN_ARM64 + masm.Pop(scratch); + masm.moveToStackPtr(scratch); +#endif + + masm.setFramePushed(framePushed); + masm.PopRegsInMask(AllAllocatableRegs); + + GenerateExitEpilogue(masm, 0, ExitReason::Fixed::DebugTrap, offsets); + + return FinishOffsets(masm, offsets); +} + +bool wasm::GenerateEntryStubs(MacroAssembler& masm, size_t funcExportIndex, + const FuncExport& fe, const FuncType& funcType, + const Maybe& callee, bool isAsmJS, + CodeRangeVector* codeRanges) { + MOZ_ASSERT(!callee == fe.hasEagerStubs()); + MOZ_ASSERT_IF(isAsmJS, fe.hasEagerStubs()); + + Offsets offsets; + if (!GenerateInterpEntry(masm, fe, funcType, callee, &offsets)) { + return false; + } + if (!codeRanges->emplaceBack(CodeRange::InterpEntry, fe.funcIndex(), + offsets)) { + return false; + } + + if (isAsmJS || !funcType.canHaveJitEntry()) { + return true; + } + + CallableOffsets jitOffsets; + if (!GenerateJitEntry(masm, funcExportIndex, fe, funcType, callee, + &jitOffsets)) { + return false; + } + if (!codeRanges->emplaceBack(CodeRange::JitEntry, fe.funcIndex(), + jitOffsets)) { + return false; + } + + return true; +} + +bool wasm::GenerateProvisionalLazyJitEntryStub(MacroAssembler& masm, + Offsets* offsets) { + AssertExpectedSP(masm); + masm.setFramePushed(0); + offsets->begin = masm.currentOffset(); + +#ifdef JS_CODEGEN_ARM64 + // Unaligned ABI calls require SP+PSP, but our mode here is SP-only + masm.SetStackPointer64(PseudoStackPointer64); + masm.Mov(PseudoStackPointer64, sp); +#endif + +#ifdef JS_USE_LINK_REGISTER + masm.pushReturnAddress(); +#endif + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); + Register temp = regs.takeAny(); + + using Fn = void* (*)(); + masm.setupUnalignedABICall(temp); + masm.callWithABI( + MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame); + +#ifdef JS_USE_LINK_REGISTER + masm.popReturnAddress(); +#endif + + masm.jump(ReturnReg); + +#ifdef JS_CODEGEN_ARM64 + // Undo the SP+PSP mode + masm.SetStackPointer64(sp); +#endif + + return FinishOffsets(masm, offsets); +} + +bool wasm::GenerateStubs(const ModuleEnvironment& env, + const FuncImportVector& imports, + const FuncExportVector& exports, CompiledCode* code) { + LifoAlloc lifo(STUBS_LIFO_DEFAULT_CHUNK_SIZE); + TempAllocator alloc(&lifo); + JitContext jcx; + WasmMacroAssembler masm(alloc, env); + AutoCreatedBy acb(masm, "wasm::GenerateStubs"); + + // Swap in already-allocated empty vectors to avoid malloc/free. + if (!code->swap(masm)) { + return false; + } + + Label throwLabel; + + JitSpew(JitSpew_Codegen, "# Emitting wasm import stubs"); + + for (uint32_t funcIndex = 0; funcIndex < imports.length(); funcIndex++) { + const FuncImport& fi = imports[funcIndex]; + const FuncType& funcType = *env.funcs[funcIndex].type; + + CallableOffsets interpOffsets; + if (!GenerateImportInterpExit(masm, fi, funcType, funcIndex, &throwLabel, + &interpOffsets)) { + return false; + } + if (!code->codeRanges.emplaceBack(CodeRange::ImportInterpExit, funcIndex, + interpOffsets)) { + return false; + } + + // Skip if the function does not have a signature that allows for a JIT + // exit. + if (!funcType.canHaveJitExit()) { + continue; + } + + CallableOffsets jitOffsets; + if (!GenerateImportJitExit(masm, fi, funcType, funcIndex, &throwLabel, + &jitOffsets)) { + return false; + } + if (!code->codeRanges.emplaceBack(CodeRange::ImportJitExit, funcIndex, + jitOffsets)) { + return false; + } + } + + JitSpew(JitSpew_Codegen, "# Emitting wasm export stubs"); + + Maybe noAbsolute; + for (size_t i = 0; i < exports.length(); i++) { + const FuncExport& fe = exports[i]; + const FuncType& funcType = (*env.types)[fe.typeIndex()].funcType(); + if (!fe.hasEagerStubs()) { + continue; + } + if (!GenerateEntryStubs(masm, i, fe, funcType, noAbsolute, env.isAsmJS(), + &code->codeRanges)) { + return false; + } + } + + JitSpew(JitSpew_Codegen, "# Emitting wasm exit stubs"); + + Offsets offsets; + + if (!GenerateTrapExit(masm, &throwLabel, &offsets)) { + return false; + } + if (!code->codeRanges.emplaceBack(CodeRange::TrapExit, offsets)) { + return false; + } + + CallableOffsets callableOffsets; + if (!GenerateDebugTrapStub(masm, &throwLabel, &callableOffsets)) { + return false; + } + if (!code->codeRanges.emplaceBack(CodeRange::DebugTrap, callableOffsets)) { + return false; + } + + if (!GenerateThrowStub(masm, &throwLabel, &offsets)) { + return false; + } + if (!code->codeRanges.emplaceBack(CodeRange::Throw, offsets)) { + return false; + } + + masm.finish(); + if (masm.oom()) { + return false; + } + + return code->swap(masm); +} diff --git a/js/src/wasm/WasmStubs.h b/js/src/wasm/WasmStubs.h new file mode 100644 index 0000000000..6a6622fdf2 --- /dev/null +++ b/js/src/wasm/WasmStubs.h @@ -0,0 +1,370 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_stubs_h +#define wasm_stubs_h + +#include "wasm/WasmFrameIter.h" // js::wasm::ExitReason +#include "wasm/WasmGenerator.h" +#include "wasm/WasmOpIter.h" + +namespace js { +namespace wasm { + +using jit::FloatRegister; +using jit::Register; +using jit::Register64; + +// ValType and location for a single result: either in a register or on the +// stack. + +class ABIResult { + ValType type_; + enum class Location { Gpr, Gpr64, Fpr, Stack } loc_; + union { + Register gpr_; + Register64 gpr64_; + FloatRegister fpr_; + uint32_t stackOffset_; + }; + + void validate() { +#ifdef DEBUG + if (onStack()) { + return; + } + MOZ_ASSERT(inRegister()); + switch (type_.kind()) { + case ValType::I32: + MOZ_ASSERT(loc_ == Location::Gpr); + break; + case ValType::I64: + MOZ_ASSERT(loc_ == Location::Gpr64); + break; + case ValType::F32: + case ValType::F64: + MOZ_ASSERT(loc_ == Location::Fpr); + break; + case ValType::Ref: + MOZ_ASSERT(loc_ == Location::Gpr); + break; + case ValType::V128: + MOZ_ASSERT(loc_ == Location::Fpr); + break; + } +#endif + } + + friend class ABIResultIter; + ABIResult() {} + + public: + // Sizes of items in the stack area. + // + // The size values come from the implementations of Push() in + // MacroAssembler-x86-shared.cpp and MacroAssembler-arm-shared.cpp, and from + // VFPRegister::size() in Architecture-arm.h. + // + // On ARM unlike on x86 we push a single for float. + + static constexpr size_t StackSizeOfPtr = sizeof(intptr_t); + static constexpr size_t StackSizeOfInt32 = StackSizeOfPtr; + static constexpr size_t StackSizeOfInt64 = sizeof(int64_t); +#if defined(JS_CODEGEN_ARM) + static constexpr size_t StackSizeOfFloat = sizeof(float); +#else + static constexpr size_t StackSizeOfFloat = sizeof(double); +#endif + static constexpr size_t StackSizeOfDouble = sizeof(double); +#ifdef ENABLE_WASM_SIMD + static constexpr size_t StackSizeOfV128 = sizeof(V128); +#endif + + ABIResult(ValType type, Register gpr) + : type_(type), loc_(Location::Gpr), gpr_(gpr) { + validate(); + } + ABIResult(ValType type, Register64 gpr64) + : type_(type), loc_(Location::Gpr64), gpr64_(gpr64) { + validate(); + } + ABIResult(ValType type, FloatRegister fpr) + : type_(type), loc_(Location::Fpr), fpr_(fpr) { + validate(); + } + ABIResult(ValType type, uint32_t stackOffset) + : type_(type), loc_(Location::Stack), stackOffset_(stackOffset) { + validate(); + } + + ValType type() const { return type_; } + bool onStack() const { return loc_ == Location::Stack; } + bool inRegister() const { return !onStack(); } + Register gpr() const { + MOZ_ASSERT(loc_ == Location::Gpr); + return gpr_; + } + Register64 gpr64() const { + MOZ_ASSERT(loc_ == Location::Gpr64); + return gpr64_; + } + FloatRegister fpr() const { + MOZ_ASSERT(loc_ == Location::Fpr); + return fpr_; + } + // Offset from SP. + uint32_t stackOffset() const { + MOZ_ASSERT(loc_ == Location::Stack); + return stackOffset_; + } + uint32_t size() const; +}; + +// Just as WebAssembly functions can take multiple arguments, they can also +// return multiple results. As with a call, a limited number of results will be +// located in registers, and the rest will be stored in a stack area. The +// |ABIResultIter| computes result locations, given a |ResultType|. +// +// Recall that a |ResultType| represents a sequence of value types t1..tN, +// indexed from 1 to N. In principle it doesn't matter how we decide which +// results get to be in registers and which go to the stack. To better +// harmonize with WebAssembly's abstract stack machine, whose properties are +// taken advantage of by the baseline compiler, our strategy is to start +// allocating result locations in "reverse" order: from result N down to 1. +// +// If a result with index I is in a register, then all results with index J > I +// are also in registers. If a result I is on the stack, then all results with +// index K < I are also on the stack, farther away from the stack pointer than +// result I. +// +// Currently only a single result is ever stored in a register, though this may +// change in the future on register-rich platforms. +// +// NB: The baseline compiler also uses thie ABI for locations of block +// parameters and return values, within individual WebAssembly functions. + +class ABIResultIter { + ResultType type_; + uint32_t count_; + uint32_t index_; + uint32_t nextStackOffset_; + enum { Next, Prev } direction_; + ABIResult cur_; + + void settleRegister(ValType type); + void settleNext(); + void settlePrev(); + + public: + explicit ABIResultIter(const ResultType& type) + : type_(type), count_(type.length()) { + reset(); + } + + void reset() { + index_ = nextStackOffset_ = 0; + direction_ = Next; + if (!done()) { + settleNext(); + } + } + bool done() const { return index_ == count_; } + uint32_t index() const { return index_; } + uint32_t count() const { return count_; } + uint32_t remaining() const { return count_ - index_; } + void switchToNext() { + MOZ_ASSERT(direction_ == Prev); + if (!done() && cur().onStack()) { + nextStackOffset_ += cur().size(); + } + index_ = count_ - index_; + direction_ = Next; + if (!done()) { + settleNext(); + } + } + void switchToPrev() { + MOZ_ASSERT(direction_ == Next); + if (!done() && cur().onStack()) { + nextStackOffset_ -= cur().size(); + } + index_ = count_ - index_; + direction_ = Prev; + if (!done()) settlePrev(); + } + void next() { + MOZ_ASSERT(direction_ == Next); + MOZ_ASSERT(!done()); + index_++; + if (!done()) { + settleNext(); + } + } + void prev() { + MOZ_ASSERT(direction_ == Prev); + MOZ_ASSERT(!done()); + index_++; + if (!done()) { + settlePrev(); + } + } + const ABIResult& cur() const { + MOZ_ASSERT(!done()); + return cur_; + } + + uint32_t stackBytesConsumedSoFar() const { return nextStackOffset_; } + + static inline bool HasStackResults(const ResultType& type) { + return type.length() > MaxRegisterResults; + } + + static uint32_t MeasureStackBytes(const ResultType& type) { + if (!HasStackResults(type)) { + return 0; + } + ABIResultIter iter(type); + while (!iter.done()) { + iter.next(); + } + return iter.stackBytesConsumedSoFar(); + } +}; + +extern bool GenerateBuiltinThunk(jit::MacroAssembler& masm, + jit::ABIFunctionType abiType, + ExitReason exitReason, void* funcPtr, + CallableOffsets* offsets); + +extern bool GenerateImportFunctions(const ModuleEnvironment& env, + const FuncImportVector& imports, + CompiledCode* code); + +extern bool GenerateStubs(const ModuleEnvironment& env, + const FuncImportVector& imports, + const FuncExportVector& exports, CompiledCode* code); + +extern bool GenerateEntryStubs(jit::MacroAssembler& masm, + size_t funcExportIndex, const FuncExport& fe, + const FuncType& funcType, + const Maybe& callee, bool isAsmJS, + CodeRangeVector* codeRanges); + +extern void GenerateTrapExitRegisterOffsets(jit::RegisterOffsets* offsets, + size_t* numWords); + +extern bool GenerateProvisionalLazyJitEntryStub(jit::MacroAssembler& masm, + Offsets* offsets); + +// A value that is written into the trap exit frame, which is useful for +// cross-checking during garbage collection. +static constexpr uintptr_t TrapExitDummyValue = 1337; + +// And its offset, in words, down from the highest-addressed word of the trap +// exit frame. The value is written into the frame using WasmPush. In the +// case where WasmPush allocates more than one word, the value will therefore +// be written at the lowest-addressed word. +#ifdef JS_CODEGEN_ARM64 +static constexpr size_t TrapExitDummyValueOffsetFromTop = 1; +#else +static constexpr size_t TrapExitDummyValueOffsetFromTop = 0; +#endif + +// An argument that will end up on the stack according to the system ABI, to be +// passed to GenerateDirectCallFromJit. Since the direct JIT call creates its +// own frame, it is its responsibility to put stack arguments to their expected +// locations; so the caller of GenerateDirectCallFromJit can put them anywhere. + +class JitCallStackArg { + public: + enum class Tag { + Imm32, + GPR, + FPU, + Address, + Undefined, + }; + + private: + Tag tag_; + union U { + int32_t imm32_; + jit::Register gpr_; + jit::FloatRegister fpu_; + jit::Address addr_; + U() {} + } arg; + + public: + JitCallStackArg() : tag_(Tag::Undefined) {} + explicit JitCallStackArg(int32_t imm32) : tag_(Tag::Imm32) { + arg.imm32_ = imm32; + } + explicit JitCallStackArg(jit::Register gpr) : tag_(Tag::GPR) { + arg.gpr_ = gpr; + } + explicit JitCallStackArg(jit::FloatRegister fpu) : tag_(Tag::FPU) { + new (&arg) jit::FloatRegister(fpu); + } + explicit JitCallStackArg(const jit::Address& addr) : tag_(Tag::Address) { + new (&arg) jit::Address(addr); + } + + Tag tag() const { return tag_; } + int32_t imm32() const { + MOZ_ASSERT(tag_ == Tag::Imm32); + return arg.imm32_; + } + jit::Register gpr() const { + MOZ_ASSERT(tag_ == Tag::GPR); + return arg.gpr_; + } + jit::FloatRegister fpu() const { + MOZ_ASSERT(tag_ == Tag::FPU); + return arg.fpu_; + } + const jit::Address& addr() const { + MOZ_ASSERT(tag_ == Tag::Address); + return arg.addr_; + } +}; + +using JitCallStackArgVector = Vector; + +// Generates an inline wasm call (during jit compilation) to a specific wasm +// function (as specifed by the given FuncExport). +// This call doesn't go through a wasm entry, but rather creates its own +// inlined exit frame. +// Assumes: +// - all the registers have been preserved by the caller, +// - all arguments passed in registers have been set up at the expected +// locations, +// - all arguments passed on stack slot are alive as defined by a corresponding +// JitCallStackArg. + +extern void GenerateDirectCallFromJit(jit::MacroAssembler& masm, + const FuncExport& fe, + const Instance& inst, + const JitCallStackArgVector& stackArgs, + jit::Register scratch, + uint32_t* callOffset); + +} // namespace wasm +} // namespace js + +#endif // wasm_stubs_h diff --git a/js/src/wasm/WasmTable.cpp b/js/src/wasm/WasmTable.cpp new file mode 100644 index 0000000000..076a344e29 --- /dev/null +++ b/js/src/wasm/WasmTable.cpp @@ -0,0 +1,473 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmTable.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/PodOperations.h" + +#include "vm/JSContext.h" +#include "vm/Realm.h" +#include "wasm/WasmInstance.h" +#include "wasm/WasmJS.h" + +#include "gc/StableCellHasher-inl.h" +#include "wasm/WasmInstance-inl.h" + +using namespace js; +using namespace js::wasm; +using mozilla::CheckedInt; +using mozilla::PodZero; + +Table::Table(JSContext* cx, const TableDesc& desc, + Handle maybeObject, FuncRefVector&& functions) + : maybeObject_(maybeObject), + observers_(cx->zone()), + functions_(std::move(functions)), + elemType_(desc.elemType), + isAsmJS_(desc.isAsmJS), + length_(desc.initialLength), + maximum_(desc.maximumLength) { + MOZ_ASSERT(repr() == TableRepr::Func); +} + +Table::Table(JSContext* cx, const TableDesc& desc, + Handle maybeObject, TableAnyRefVector&& objects) + : maybeObject_(maybeObject), + observers_(cx->zone()), + objects_(std::move(objects)), + elemType_(desc.elemType), + isAsmJS_(desc.isAsmJS), + length_(desc.initialLength), + maximum_(desc.maximumLength) { + MOZ_ASSERT(repr() == TableRepr::Ref); +} + +/* static */ +SharedTable Table::create(JSContext* cx, const TableDesc& desc, + Handle maybeObject) { + // Tables are initialized with init_expr values at Instance::init or + // WasmTableObject::create. + + switch (desc.elemType.tableRepr()) { + case TableRepr::Func: { + FuncRefVector functions; + if (!functions.resize(desc.initialLength)) { + ReportOutOfMemory(cx); + return nullptr; + } + return SharedTable( + cx->new_
(cx, desc, maybeObject, std::move(functions))); + } + case TableRepr::Ref: { + TableAnyRefVector objects; + if (!objects.resize(desc.initialLength)) { + ReportOutOfMemory(cx); + return nullptr; + } + return SharedTable( + cx->new_
(cx, desc, maybeObject, std::move(objects))); + } + } + MOZ_CRASH("switch is exhaustive"); +} + +void Table::tracePrivate(JSTracer* trc) { + // If this table has a WasmTableObject, then this method is only called by + // WasmTableObject's trace hook so maybeObject_ must already be marked. + // TraceEdge is called so that the pointer can be updated during a moving + // GC. + TraceNullableEdge(trc, &maybeObject_, "wasm table object"); + + switch (repr()) { + case TableRepr::Func: { + if (isAsmJS_) { +#ifdef DEBUG + for (uint32_t i = 0; i < length_; i++) { + MOZ_ASSERT(!functions_[i].instance); + } +#endif + break; + } + + for (uint32_t i = 0; i < length_; i++) { + if (functions_[i].instance) { + wasm::TraceInstanceEdge(trc, functions_[i].instance, + "wasm table instance"); + } else { + MOZ_ASSERT(!functions_[i].code); + } + } + break; + } + case TableRepr::Ref: { + objects_.trace(trc); + break; + } + } +} + +void Table::trace(JSTracer* trc) { + // The trace hook of WasmTableObject will call Table::tracePrivate at + // which point we can mark the rest of the children. If there is no + // WasmTableObject, call Table::tracePrivate directly. Redirecting through + // the WasmTableObject avoids marking the entire Table on each incoming + // edge (once per dependent Instance). + if (maybeObject_) { + TraceEdge(trc, &maybeObject_, "wasm table object"); + } else { + tracePrivate(trc); + } +} + +uint8_t* Table::instanceElements() const { + if (repr() == TableRepr::Ref) { + return (uint8_t*)objects_.begin(); + } + return (uint8_t*)functions_.begin(); +} + +const FunctionTableElem& Table::getFuncRef(uint32_t index) const { + MOZ_ASSERT(isFunction()); + return functions_[index]; +} + +bool Table::getFuncRef(JSContext* cx, uint32_t index, + MutableHandleFunction fun) const { + MOZ_ASSERT(isFunction()); + + const FunctionTableElem& elem = getFuncRef(index); + if (!elem.code) { + fun.set(nullptr); + return true; + } + + Instance& instance = *elem.instance; + const CodeRange& codeRange = *instance.code().lookupFuncRange(elem.code); + + Rooted instanceObj(cx, instance.object()); + return instanceObj->getExportedFunction(cx, instanceObj, + codeRange.funcIndex(), fun); +} + +void Table::setFuncRef(uint32_t index, void* code, Instance* instance) { + MOZ_ASSERT(isFunction()); + + FunctionTableElem& elem = functions_[index]; + if (elem.instance) { + gc::PreWriteBarrier(elem.instance->objectUnbarriered()); + } + + if (!isAsmJS_) { + elem.code = code; + elem.instance = instance; + MOZ_ASSERT(elem.instance->objectUnbarriered()->isTenured(), + "no postWriteBarrier (Table::set)"); + } else { + elem.code = code; + elem.instance = nullptr; + } +} + +void Table::fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref, + JSContext* cx) { + MOZ_ASSERT(isFunction()); + + if (ref.isNull()) { + for (uint32_t i = index, end = index + fillCount; i != end; i++) { + setNull(i); + } + return; + } + + RootedFunction fun(cx, ref.asJSFunction()); + MOZ_RELEASE_ASSERT(IsWasmExportedFunction(fun)); + + Rooted instanceObj( + cx, ExportedFunctionToInstanceObject(fun)); + uint32_t funcIndex = ExportedFunctionToFuncIndex(fun); + +#ifdef DEBUG + RootedFunction f(cx); + MOZ_ASSERT(instanceObj->getExportedFunction(cx, instanceObj, funcIndex, &f)); + MOZ_ASSERT(fun == f); +#endif + + Instance& instance = instanceObj->instance(); + Tier tier = instance.code().bestTier(); + const MetadataTier& metadata = instance.metadata(tier); + const CodeRange& codeRange = + metadata.codeRange(metadata.lookupFuncExport(funcIndex)); + void* code = instance.codeBase(tier) + codeRange.funcCheckedCallEntry(); + for (uint32_t i = index, end = index + fillCount; i != end; i++) { + setFuncRef(i, code, &instance); + } +} + +AnyRef Table::getAnyRef(uint32_t index) const { + MOZ_ASSERT(!isFunction()); + // TODO/AnyRef-boxing: With boxed immediates and strings, the write barrier + // is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + return AnyRef::fromJSObject(objects_[index]); +} + +void Table::fillAnyRef(uint32_t index, uint32_t fillCount, AnyRef ref) { + MOZ_ASSERT(!isFunction()); + // TODO/AnyRef-boxing: With boxed immediates and strings, the write barrier + // is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + for (uint32_t i = index, end = index + fillCount; i != end; i++) { + objects_[i] = ref.asJSObject(); + } +} + +bool Table::getValue(JSContext* cx, uint32_t index, + MutableHandleValue result) const { + switch (repr()) { + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(!isAsmJS()); + RootedFunction fun(cx); + if (!getFuncRef(cx, index, &fun)) { + return false; + } + result.setObjectOrNull(fun); + return true; + } + case TableRepr::Ref: { + if (!ValType(elemType_).isExposable()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; + } + return ToJSValue(cx, &objects_[index], ValType(elemType_), result); + } + default: + MOZ_CRASH(); + } +} + +void Table::setNull(uint32_t index) { + switch (repr()) { + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(!isAsmJS_); + FunctionTableElem& elem = functions_[index]; + if (elem.instance) { + gc::PreWriteBarrier(elem.instance->objectUnbarriered()); + } + + elem.code = nullptr; + elem.instance = nullptr; + break; + } + case TableRepr::Ref: { + fillAnyRef(index, 1, AnyRef::null()); + break; + } + } +} + +bool Table::copy(JSContext* cx, const Table& srcTable, uint32_t dstIndex, + uint32_t srcIndex) { + MOZ_RELEASE_ASSERT(!srcTable.isAsmJS_); + switch (repr()) { + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(elemType().isFuncHierarchy() && + srcTable.elemType().isFuncHierarchy()); + FunctionTableElem& dst = functions_[dstIndex]; + if (dst.instance) { + gc::PreWriteBarrier(dst.instance->objectUnbarriered()); + } + + const FunctionTableElem& src = srcTable.functions_[srcIndex]; + dst.code = src.code; + dst.instance = src.instance; + + if (dst.instance) { + MOZ_ASSERT(dst.code); + MOZ_ASSERT(dst.instance->objectUnbarriered()->isTenured(), + "no postWriteBarrier (Table::copy)"); + } else { + MOZ_ASSERT(!dst.code); + } + break; + } + case TableRepr::Ref: { + switch (srcTable.repr()) { + case TableRepr::Ref: { + fillAnyRef(dstIndex, 1, srcTable.getAnyRef(srcIndex)); + break; + } + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(srcTable.elemType().isFuncHierarchy()); + // Upcast. + RootedFunction fun(cx); + if (!srcTable.getFuncRef(cx, srcIndex, &fun)) { + // OOM, so just pass it on. + return false; + } + fillAnyRef(dstIndex, 1, AnyRef::fromJSObject(fun)); + break; + } + } + break; + } + } + return true; +} + +uint32_t Table::grow(uint32_t delta) { + // This isn't just an optimization: movingGrowable() assumes that + // onMovingGrowTable does not fire when length == maximum. + if (!delta) { + return length_; + } + + uint32_t oldLength = length_; + + CheckedInt newLength = oldLength; + newLength += delta; + if (!newLength.isValid() || newLength.value() > MaxTableLength) { + return -1; + } + + if (maximum_ && newLength.value() > maximum_.value()) { + return -1; + } + + MOZ_ASSERT(movingGrowable()); + + switch (repr()) { + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(!isAsmJS_); + if (!functions_.resize(newLength.value())) { + return -1; + } + break; + } + case TableRepr::Ref: { + if (!objects_.resize(newLength.value())) { + return -1; + } + break; + } + } + + if (auto* object = maybeObject_.unbarrieredGet()) { + RemoveCellMemory(object, gcMallocBytes(), MemoryUse::WasmTableTable); + } + + length_ = newLength.value(); + + if (auto* object = maybeObject_.unbarrieredGet()) { + AddCellMemory(object, gcMallocBytes(), MemoryUse::WasmTableTable); + } + + for (InstanceSet::Range r = observers_.all(); !r.empty(); r.popFront()) { + r.front()->instance().onMovingGrowTable(this); + } + + return oldLength; +} + +bool Table::movingGrowable() const { + return !maximum_ || length_ < maximum_.value(); +} + +bool Table::addMovingGrowObserver(JSContext* cx, WasmInstanceObject* instance) { + MOZ_ASSERT(movingGrowable()); + + // A table can be imported multiple times into an instance, but we only + // register the instance as an observer once. + + if (!observers_.put(instance)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +void Table::fillUninitialized(uint32_t index, uint32_t fillCount, + HandleAnyRef ref, JSContext* cx) { +#ifdef DEBUG + assertRangeNull(index, fillCount); +#endif // DEBUG + switch (repr()) { + case TableRepr::Func: { + MOZ_RELEASE_ASSERT(!isAsmJS_); + fillFuncRef(index, fillCount, FuncRef::fromAnyRefUnchecked(ref), cx); + break; + } + case TableRepr::Ref: { + fillAnyRef(index, fillCount, ref); + break; + } + } +} + +#ifdef DEBUG +void Table::assertRangeNull(uint32_t index, uint32_t length) const { + switch (repr()) { + case TableRepr::Func: + for (uint32_t i = index; i < index + length; i++) { + MOZ_ASSERT(getFuncRef(i).instance == nullptr); + MOZ_ASSERT(getFuncRef(i).code == nullptr); + } + break; + case TableRepr::Ref: + for (uint32_t i = index; i < index + length; i++) { + MOZ_ASSERT(getAnyRef(i).isNull()); + } + break; + } +} + +void Table::assertRangeNotNull(uint32_t index, uint32_t length) const { + switch (repr()) { + case TableRepr::Func: + for (uint32_t i = index; i < index + length; i++) { + MOZ_ASSERT_IF(!isAsmJS_, getFuncRef(i).instance != nullptr); + MOZ_ASSERT(getFuncRef(i).code != nullptr); + } + break; + case TableRepr::Ref: + for (uint32_t i = index; i < index + length; i++) { + MOZ_ASSERT(!getAnyRef(i).isNull()); + } + break; + } +} +#endif // DEBUG + +size_t Table::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + if (isFunction()) { + return functions_.sizeOfExcludingThis(mallocSizeOf); + } + return objects_.sizeOfExcludingThis(mallocSizeOf); +} + +size_t Table::gcMallocBytes() const { + size_t size = sizeof(*this); + if (isFunction()) { + size += length() * sizeof(FunctionTableElem); + } else { + size += length() * sizeof(TableAnyRefVector::ElementType); + } + return size; +} diff --git a/js/src/wasm/WasmTable.h b/js/src/wasm/WasmTable.h new file mode 100644 index 0000000000..80cc4d3bdf --- /dev/null +++ b/js/src/wasm/WasmTable.h @@ -0,0 +1,139 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_table_h +#define wasm_table_h + +#include "gc/Policy.h" +#include "wasm/WasmCode.h" + +namespace js { +namespace wasm { + +// A Table is an indexable array of opaque values. Tables are first-class +// stateful objects exposed to WebAssembly. asm.js also uses Tables to represent +// its homogeneous function-pointer tables. +// +// A table of FuncRef holds FunctionTableElems, which are (code*,instance*) +// pairs, where the instance must be traced. +// +// A table of AnyRef holds JSObject pointers, which must be traced. + +// TODO/AnyRef-boxing: With boxed immediates and strings, JSObject* is no longer +// the most appropriate representation for Cell::anyref. +STATIC_ASSERT_ANYREF_IS_JSOBJECT; + +using TableAnyRefVector = GCVector, 0, SystemAllocPolicy>; + +class Table : public ShareableBase
{ + using InstanceSet = JS::WeakCache, + StableCellHasher>, SystemAllocPolicy>>; + using FuncRefVector = Vector; + + WeakHeapPtr maybeObject_; + InstanceSet observers_; + FuncRefVector functions_; // either functions_ has data + TableAnyRefVector objects_; // or objects_, but not both + const RefType elemType_; + const bool isAsmJS_; + uint32_t length_; + const Maybe maximum_; + + template + friend struct js::MallocProvider; + Table(JSContext* cx, const TableDesc& desc, + Handle maybeObject, FuncRefVector&& functions); + Table(JSContext* cx, const TableDesc& desc, + Handle maybeObject, TableAnyRefVector&& objects); + + void tracePrivate(JSTracer* trc); + friend class js::WasmTableObject; + + public: + static RefPtr
create(JSContext* cx, const TableDesc& desc, + Handle maybeObject); + void trace(JSTracer* trc); + + RefType elemType() const { return elemType_; } + TableRepr repr() const { return elemType_.tableRepr(); } + + bool isAsmJS() const { + MOZ_ASSERT(elemType_.isFuncHierarchy()); + return isAsmJS_; + } + + bool isFunction() const { return elemType().isFuncHierarchy(); } + uint32_t length() const { return length_; } + Maybe maximum() const { return maximum_; } + + // Raw pointer to the table for use in TableInstanceData. + uint8_t* instanceElements() const; + + // set/get/fillFuncRef is allowed only on table-of-funcref. + // get/fillAnyRef is allowed only on table-of-anyref. + // setNull is allowed on either. + + const FunctionTableElem& getFuncRef(uint32_t index) const; + [[nodiscard]] bool getFuncRef(JSContext* cx, uint32_t index, + MutableHandleFunction fun) const; + void setFuncRef(uint32_t index, void* code, Instance* instance); + void fillFuncRef(uint32_t index, uint32_t fillCount, FuncRef ref, + JSContext* cx); + + AnyRef getAnyRef(uint32_t index) const; + void fillAnyRef(uint32_t index, uint32_t fillCount, AnyRef ref); + + // Get the element at index and convert it to a JS value. + [[nodiscard]] bool getValue(JSContext* cx, uint32_t index, + MutableHandleValue result) const; + + void setNull(uint32_t index); + + // Copy entry from |srcTable| at |srcIndex| to this table at |dstIndex|. Used + // by table.copy. May OOM if it needs to box up a function during an upcast. + [[nodiscard]] bool copy(JSContext* cx, const Table& srcTable, + uint32_t dstIndex, uint32_t srcIndex); + + // grow() returns (uint32_t)-1 if it could not grow. + [[nodiscard]] uint32_t grow(uint32_t delta); + [[nodiscard]] bool movingGrowable() const; + [[nodiscard]] bool addMovingGrowObserver(JSContext* cx, + WasmInstanceObject* instance); + + void fillUninitialized(uint32_t index, uint32_t fillCount, HandleAnyRef ref, + JSContext* cx); +#ifdef DEBUG + void assertRangeNull(uint32_t index, uint32_t length) const; + void assertRangeNotNull(uint32_t index, uint32_t length) const; +#endif // DEBUG + + // about:memory reporting: + + size_t sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const; + + size_t gcMallocBytes() const; +}; + +using SharedTable = RefPtr
; +using SharedTableVector = Vector; + +} // namespace wasm +} // namespace js + +#endif // wasm_table_h diff --git a/js/src/wasm/WasmTypeDecls.h b/js/src/wasm/WasmTypeDecls.h new file mode 100644 index 0000000000..71ea66845c --- /dev/null +++ b/js/src/wasm/WasmTypeDecls.h @@ -0,0 +1,99 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_type_decls_h +#define wasm_type_decls_h + +#include "NamespaceImports.h" +#include "gc/Barrier.h" +#include "js/GCVector.h" +#include "js/HashTable.h" +#include "js/RootingAPI.h" +#include "js/UniquePtr.h" +#include "js/Utility.h" +#include "js/Vector.h" + +namespace js { + +using JSObjectVector = GCVector; + +class WasmMemoryObject; +class WasmModuleObject; +class WasmInstanceObject; +class WasmTableObject; +class WasmGlobalObject; +class WasmTagObject; +class WasmExceptionObject; + +using WasmInstanceObjectVector = GCVector; +using WasmTableObjectVector = GCVector; +using WasmGlobalObjectVector = + GCVector; +using WasmTagObjectVector = GCVector; + +namespace wasm { + +struct ModuleEnvironment; +class CodeRange; +class CodeTier; +class ModuleSegment; +struct Metadata; +struct MetadataTier; +class Decoder; +class GeneratedSourceMap; +class Instance; +class Module; + +class Code; +using SharedCode = RefPtr; +using MutableCode = RefPtr; + +class Table; +using SharedTable = RefPtr
; +using SharedTableVector = Vector; + +class DebugState; +using UniqueDebugState = UniquePtr; + +struct DataSegment; +using MutableDataSegment = RefPtr; +using SharedDataSegment = RefPtr; +using DataSegmentVector = Vector; + +struct ElemSegment; +using MutableElemSegment = RefPtr; +using SharedElemSegment = RefPtr; +using ElemSegmentVector = Vector; + +class Val; +using ValVector = GCVector; + +// Uint32Vector has initial size 8 on the basis that the dominant use cases +// (line numbers and control stacks) tend to have a small but nonzero number +// of elements. +using Uint32Vector = Vector; + +using Bytes = Vector; +using UTF8Bytes = Vector; +using InstanceVector = Vector; +using UniqueCharsVector = Vector; + +} // namespace wasm +} // namespace js + +#endif // wasm_type_decls_h diff --git a/js/src/wasm/WasmTypeDef.cpp b/js/src/wasm/WasmTypeDef.cpp new file mode 100644 index 0000000000..8aad9fea31 --- /dev/null +++ b/js/src/wasm/WasmTypeDef.cpp @@ -0,0 +1,550 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2015 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmTypeDef.h" + +#include "mozilla/MathAlgorithms.h" + +#include "jit/JitOptions.h" +#include "js/friend/ErrorMessages.h" // JSMSG_* +#include "js/HashTable.h" +#include "js/Printf.h" +#include "js/Value.h" +#include "threading/ExclusiveData.h" +#include "vm/Runtime.h" +#include "vm/StringType.h" +#include "wasm/WasmCodegenConstants.h" +#include "wasm/WasmGcObject.h" +#include "wasm/WasmJS.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::CheckedUint32; +using mozilla::IsPowerOfTwo; + +// [SMDOC] Immediate type signature encoding +// +// call_indirect requires a signature check to ensure the dynamic callee type +// matches the static specified callee type. This involves comparing whether +// the two function types are equal. We canonicalize function types so that +// comparing the pointers of the types will indicate if they're equal. The +// canonicalized function types are loaded from the instance at runtime. +// +// For the common case of simple/small function types, we can avoid the cost +// of loading the function type pointers from the instance by having an +// alternate 'immediate' form that encodes a function type in a constant. +// We encode the function types such that bitwise equality implies the original +// function types were equal. We use a tag bit such that if one of the types +// is a pointer and the other an immediate, they will not compare as equal. +// +// The encoding is optimized for common function types that have at most one +// result and an arbitrary amount of arguments. +// +// [ +// 1 bit : tag (always 1), +// 1 bit : numResults, +// 3 bits : numArgs, +// numResults * 3 bits : results, +// numArgs * 3 bits : args +// ] +// (lsb -> msb order) +// +// Any function type that cannot be encoded in the above format is falls back +// to the pointer representation. +// + +//========================================================================= +// ImmediateType + +// ImmediateType is 32-bits to ensure it's easy to materialize the constant +// on all platforms. +using ImmediateType = uint32_t; +static const unsigned sTotalBits = sizeof(ImmediateType) * 8; +static const unsigned sTagBits = 1; +static const unsigned sNumResultsBits = 1; +static const unsigned sNumArgsBits = 3; +static const unsigned sValTypeBits = 3; +static const unsigned sMaxValTypes = 8; + +static_assert(((1 << sNumResultsBits) - 1) + ((1 << sNumArgsBits) - 1) == + sMaxValTypes, + "sNumResultsBits, sNumArgsBits, sMaxValTypes are consistent"); + +static_assert(sTagBits + sNumResultsBits + sNumArgsBits + + sValTypeBits * sMaxValTypes <= + sTotalBits, + "have room"); + +static bool IsImmediateValType(ValType vt) { + switch (vt.kind()) { + case ValType::I32: + case ValType::I64: + case ValType::F32: + case ValType::F64: + case ValType::V128: + return true; + case ValType::Ref: + // We don't have space to encode nullability, so we optimize for + // non-nullable types. + if (!vt.isNullable()) { + return false; + } + switch (vt.refType().kind()) { + case RefType::Func: + case RefType::Extern: + case RefType::Any: + return true; + default: + return false; + } + default: + return false; + } +} + +static unsigned EncodeImmediateValType(ValType vt) { + // We have run out of bits for each type, anything new must increase the + // sValTypeBits. + static_assert(7 < (1 << sValTypeBits), "enough space for ValType kind"); + + switch (vt.kind()) { + case ValType::I32: + return 0; + case ValType::I64: + return 1; + case ValType::F32: + return 2; + case ValType::F64: + return 3; + case ValType::V128: + return 4; + case ValType::Ref: + MOZ_ASSERT(vt.isNullable()); + switch (vt.refType().kind()) { + case RefType::Func: + return 5; + case RefType::Extern: + return 6; + case RefType::Any: + return 7; + default: + MOZ_CRASH("bad RefType"); + } + default: + MOZ_CRASH("bad ValType"); + } +} + +static bool IsImmediateFuncType(const FuncType& funcType) { + const ValTypeVector& results = funcType.results(); + const ValTypeVector& args = funcType.args(); + + // Check the number of results and args fits + if (results.length() > ((1 << sNumResultsBits) - 1) || + args.length() > ((1 << sNumArgsBits) - 1)) { + return false; + } + + // Ensure every result is compatible + for (ValType v : results) { + if (!IsImmediateValType(v)) { + return false; + } + } + + // Ensure every arg is compatible + for (ValType v : args) { + if (!IsImmediateValType(v)) { + return false; + } + } + + return true; +} + +static ImmediateType EncodeNumResults(uint32_t numResults) { + MOZ_ASSERT(numResults <= (1 << sNumResultsBits) - 1); + return numResults; +} + +static ImmediateType EncodeNumArgs(uint32_t numArgs) { + MOZ_ASSERT(numArgs <= (1 << sNumArgsBits) - 1); + return numArgs; +} + +static ImmediateType EncodeImmediateFuncType(const FuncType& funcType) { + ImmediateType immediate = FuncType::ImmediateBit; + uint32_t shift = sTagBits; + + // Encode the results + immediate |= EncodeNumResults(funcType.results().length()) << shift; + shift += sNumResultsBits; + + for (ValType resultType : funcType.results()) { + immediate |= EncodeImmediateValType(resultType) << shift; + shift += sValTypeBits; + } + + // Encode the args + immediate |= EncodeNumArgs(funcType.args().length()) << shift; + shift += sNumArgsBits; + + for (ValType argType : funcType.args()) { + immediate |= EncodeImmediateValType(argType) << shift; + shift += sValTypeBits; + } + + MOZ_ASSERT(shift <= sTotalBits); + return immediate; +} + +//========================================================================= +// FuncType + +void FuncType::initImmediateTypeId() { + if (!IsImmediateFuncType(*this)) { + immediateTypeId_ = NO_IMMEDIATE_TYPE_ID; + return; + } + immediateTypeId_ = EncodeImmediateFuncType(*this); +} + +bool FuncType::canHaveJitEntry() const { + return !hasUnexposableArgOrRet() && + !temporarilyUnsupportedReftypeForEntry() && + !temporarilyUnsupportedResultCountForJitEntry() && + JitOptions.enableWasmJitEntry; +} + +bool FuncType::canHaveJitExit() const { + return !hasUnexposableArgOrRet() && !temporarilyUnsupportedReftypeForExit() && + !hasInt64Arg() && !temporarilyUnsupportedResultCountForJitExit() && + JitOptions.enableWasmJitExit; +} + +size_t FuncType::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return args_.sizeOfExcludingThis(mallocSizeOf); +} + +//========================================================================= +// StructType and StructLayout + +static inline CheckedInt32 RoundUpToAlignment(CheckedInt32 address, + uint32_t align) { + MOZ_ASSERT(IsPowerOfTwo(align)); + + // Note: Be careful to order operators such that we first make the + // value smaller and then larger, so that we don't get false + // overflow errors due to (e.g.) adding `align` and then + // subtracting `1` afterwards when merely adding `align-1` would + // not have overflowed. Note that due to the nature of two's + // complement representation, if `address` is already aligned, + // then adding `align-1` cannot itself cause an overflow. + + return ((address + (align - 1)) / align) * align; +} + +CheckedInt32 StructLayout::addField(FieldType type) { + uint32_t fieldSize = type.size(); + uint32_t fieldAlignment = type.alignmentInStruct(); + + // We have to ensure that `offset` is chosen so that no field crosses the + // inline/outline boundary. The assertions here ensure that. See comment + // on `class StructLayout` for background. + MOZ_ASSERT(fieldSize >= 1 && fieldSize <= 16); + MOZ_ASSERT((fieldSize & (fieldSize - 1)) == 0); // is a power of 2 + MOZ_ASSERT(fieldAlignment == fieldSize); // is naturally aligned + + // Alignment of the struct is the max of the alignment of its fields. + structAlignment = std::max(structAlignment, fieldAlignment); + + // Align the pointer. + CheckedInt32 offset = RoundUpToAlignment(sizeSoFar, fieldAlignment); + if (!offset.isValid()) { + return offset; + } + + // Allocate space. + sizeSoFar = offset + fieldSize; + if (!sizeSoFar.isValid()) { + return sizeSoFar; + } + + // The following should hold if the three assertions above hold. + MOZ_ASSERT(offset / 16 == (offset + fieldSize - 1) / 16); + return offset; +} + +CheckedInt32 StructLayout::close() { + return RoundUpToAlignment(sizeSoFar, structAlignment); +} + +bool StructType::init() { + StructLayout layout; + for (StructField& field : fields_) { + CheckedInt32 offset = layout.addField(field.type); + if (!offset.isValid()) { + return false; + } + field.offset = offset.value(); + if (!field.type.isRefRepr()) { + continue; + } + + bool isOutline; + uint32_t adjustedOffset; + WasmStructObject::fieldOffsetToAreaAndOffset(field.type, field.offset, + &isOutline, &adjustedOffset); + if (isOutline) { + if (!outlineTraceOffsets_.append(adjustedOffset)) { + return false; + } + } else { + if (!inlineTraceOffsets_.append(adjustedOffset)) { + return false; + } + } + } + + CheckedInt32 size = layout.close(); + if (!size.isValid()) { + return false; + } + size_ = size.value(); + + return true; +} + +size_t StructType::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return fields_.sizeOfExcludingThis(mallocSizeOf); +} + +size_t ArrayType::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return 0; +} + +size_t TypeDef::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + switch (kind_) { + case TypeDefKind::Struct: { + return structType_.sizeOfExcludingThis(mallocSizeOf); + } + case TypeDefKind::Func: { + return funcType_.sizeOfExcludingThis(mallocSizeOf); + } + case TypeDefKind::Array: { + return arrayType_.sizeOfExcludingThis(mallocSizeOf); + } + case TypeDefKind::None: { + return 0; + } + default: + break; + } + MOZ_ASSERT_UNREACHABLE(); + return 0; +} + +//========================================================================= +// SuperTypeVector + +/* static */ +size_t SuperTypeVector::offsetOfTypeDefInVector(uint32_t typeDefDepth) { + return offsetof(SuperTypeVector, types_) + sizeof(void*) * typeDefDepth; +} + +/* static */ +size_t SuperTypeVector::lengthForTypeDef(const TypeDef& typeDef) { + return std::max(uint32_t(typeDef.subTypingDepth()) + 1, + MinSuperTypeVectorLength); +} + +/* static */ +size_t SuperTypeVector::byteSizeForTypeDef(const TypeDef& typeDef) { + static_assert( + sizeof(SuperTypeVector) + sizeof(void*) * (MaxSubTypingDepth + 1) <= + UINT16_MAX, + "cannot overflow"); + return sizeof(SuperTypeVector) + (sizeof(void*) * lengthForTypeDef(typeDef)); +} + +/* static */ +const SuperTypeVector* SuperTypeVector::createMultipleForRecGroup( + RecGroup* recGroup) { + // Pre-size the amount of space needed for all the super type vectors in this + // recursion group. + CheckedUint32 totalBytes = 0; + for (uint32_t typeIndex = 0; typeIndex < recGroup->numTypes(); typeIndex++) { + totalBytes += + SuperTypeVector::byteSizeForTypeDef(recGroup->type(typeIndex)); + } + if (!totalBytes.isValid()) { + return nullptr; + } + + // Allocate the batch, and retain reference to the first one. + SuperTypeVector* firstVector = + (SuperTypeVector*)js_malloc(totalBytes.value()); + if (!firstVector) { + return nullptr; + } + + // Initialize the vectors, one by one + SuperTypeVector* currentVector = firstVector; + for (uint32_t typeIndex = 0; typeIndex < recGroup->numTypes(); typeIndex++) { + TypeDef& typeDef = recGroup->type(typeIndex); + + // Compute the size again to know where the next vector can be found. + size_t vectorByteSize = SuperTypeVector::byteSizeForTypeDef(typeDef); + + // Make the typedef and the vector point at each other. + typeDef.setSuperTypeVector(currentVector); + currentVector->setTypeDef(&typeDef); + + // Every vector stores all ancestor types and itself. + currentVector->setLength(SuperTypeVector::lengthForTypeDef(typeDef)); + + // Initialize the entries in the vector + const TypeDef* currentTypeDef = &typeDef; + for (uint32_t index = 0; index < currentVector->length(); index++) { + uint32_t reverseIndex = currentVector->length() - index - 1; + + // If this entry is required just to hit the minimum size, then + // initialize it to null. + if (reverseIndex > typeDef.subTypingDepth()) { + currentVector->setType(reverseIndex, nullptr); + continue; + } + + // Otherwise we should always be iterating at the same depth as our + // currentTypeDef. + MOZ_ASSERT(reverseIndex == currentTypeDef->subTypingDepth()); + + currentVector->setType(reverseIndex, currentTypeDef->superTypeVector()); + currentTypeDef = currentTypeDef->superTypeDef(); + } + + // There should be no more super types left over + MOZ_ASSERT(currentTypeDef == nullptr); + + // Advance to the next super type vector + currentVector = + (SuperTypeVector*)(((const char*)currentVector) + vectorByteSize); + } + + return firstVector; +} + +//========================================================================= +// TypeIdSet and TypeContext + +struct RecGroupHashPolicy { + using Lookup = const SharedRecGroup&; + + static HashNumber hash(Lookup lookup) { return lookup->hash(); } + + static bool match(const SharedRecGroup& lhs, Lookup rhs) { + return RecGroup::matches(*rhs, *lhs); + } +}; + +// A global hash set of recursion groups for use in fast type equality checks. +class TypeIdSet { + using Set = HashSet; + Set set_; + + public: + // Attempt to insert a recursion group into the set, returning an existing + // recursion group if there was one. + SharedRecGroup insert(SharedRecGroup recGroup) { + Set::AddPtr p = set_.lookupForAdd(recGroup); + if (p) { + // A canonical recursion group already existed, return it. + return *p; + } + + // Insert this recursion group into the set, and return it as the canonical + // recursion group instance. + if (!set_.add(p, recGroup)) { + return nullptr; + } + return recGroup; + } + + void purge() { + // TODO: this is not guaranteed to remove all types that are not referenced + // from outside the canonical set, as removing a type may make a previous + // type we've visited now only have one ref and be eligible to be freed. + // + // Solving this either involves iterating to a fixed point, or else a much + // more invasive change to the lifetime management of recursion groups. + for (auto iter = set_.modIter(); !iter.done(); iter.next()) { + if (iter.get()->hasOneRef()) { + iter.remove(); + } + } + } + + // Release the provided recursion group reference and remove it from the + // canonical set if it was the last reference. This is one unified method + // because we need to perform the lookup before releasing the reference, but + // need to release the reference in order to see if it was the last reference + // outside the canonical set. + void clearRecGroup(SharedRecGroup* recGroupCell) { + if (Set::Ptr p = set_.lookup(*recGroupCell)) { + *recGroupCell = nullptr; + if ((*p)->hasOneRef()) { + set_.remove(p); + } + } else { + *recGroupCell = nullptr; + } + } +}; + +ExclusiveData typeIdSet(mutexid::WasmTypeIdSet); + +void wasm::PurgeCanonicalTypes() { + ExclusiveData::Guard locked = typeIdSet.lock(); + locked->purge(); +} + +SharedRecGroup TypeContext::canonicalizeGroup(SharedRecGroup recGroup) { + ExclusiveData::Guard locked = typeIdSet.lock(); + return locked->insert(recGroup); +} + +TypeContext::~TypeContext() { + ExclusiveData::Guard locked = typeIdSet.lock(); + + // Clear out the recursion groups in this module, freeing them from the + // canonical type set if needed. + // + // We iterate backwards here so that we free every previous recursion group + // that may be referring to the current recursion group we're freeing. This + // is possible due to recursion groups being ordered. + for (int32_t groupIndex = recGroups_.length() - 1; groupIndex >= 0; + groupIndex--) { + // Try to remove this entry from the canonical set if we have the last + // strong reference. The entry may not exist if canonicalization failed + // and this type context was aborted. This will clear the reference in the + // vector. + locked->clearRecGroup(&recGroups_[groupIndex]); + } +} diff --git a/js/src/wasm/WasmTypeDef.h b/js/src/wasm/WasmTypeDef.h new file mode 100644 index 0000000000..126c04bc95 --- /dev/null +++ b/js/src/wasm/WasmTypeDef.h @@ -0,0 +1,1462 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_type_def_h +#define wasm_type_def_h + +#include "mozilla/CheckedInt.h" +#include "mozilla/HashTable.h" + +#include "js/RefCounted.h" + +#include "wasm/WasmCodegenConstants.h" +#include "wasm/WasmCompileArgs.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmUtility.h" +#include "wasm/WasmValType.h" + +namespace js { +namespace wasm { + +using mozilla::CheckedInt32; +using mozilla::MallocSizeOf; + +class RecGroup; + +//========================================================================= +// Function types + +// The FuncType class represents a WebAssembly function signature which takes a +// list of value types and returns an expression type. The engine uses two +// in-memory representations of the argument Vector's memory (when elements do +// not fit inline): normal malloc allocation (via SystemAllocPolicy) and +// allocation in a LifoAlloc (via LifoAllocPolicy). The former FuncType objects +// can have any lifetime since they own the memory. The latter FuncType objects +// must not outlive the associated LifoAlloc mark/release interval (which is +// currently the duration of module validation+compilation). Thus, long-lived +// objects like WasmModule must use malloced allocation. + +class FuncType { + ValTypeVector args_; + ValTypeVector results_; + // A packed structural type identifier for use in the call_indirect type + // check in the prologue of functions. If this function type cannot fit in + // this immediate, it will be NO_IMMEDIATE_TYPE_ID. + uint32_t immediateTypeId_; + + // This function type cannot be packed into an immediate for call_indirect + // signature checks. + static const uint32_t NO_IMMEDIATE_TYPE_ID = UINT32_MAX; + + // Entry from JS to wasm via the JIT is currently unimplemented for + // functions that return multiple values. + bool temporarilyUnsupportedResultCountForJitEntry() const { + return results().length() > MaxResultsForJitEntry; + } + // Calls out from wasm to JS that return multiple values is currently + // unsupported. + bool temporarilyUnsupportedResultCountForJitExit() const { + return results().length() > MaxResultsForJitExit; + } + // For JS->wasm jit entries, temporarily disallow certain types until the + // stubs generator is improved. + // * ref params may be nullable externrefs + // * ref results may not be type indices + // V128 types are excluded per spec but are guarded against separately. + bool temporarilyUnsupportedReftypeForEntry() const { + for (ValType arg : args()) { + if (arg.isRefType() && (!arg.isExternRef() || !arg.isNullable())) { + return true; + } + } + for (ValType result : results()) { + if (result.isTypeRef()) { + return true; + } + } + return false; + } + // For wasm->JS jit exits, temporarily disallow certain types until + // the stubs generator is improved. + // * ref results may be nullable externrefs + // Unexposable types must be guarded against separately. + bool temporarilyUnsupportedReftypeForExit() const { + for (ValType result : results()) { + if (result.isRefType() && + (!result.isExternRef() || !result.isNullable())) { + return true; + } + } + return false; + } + + void initImmediateTypeId(); + + public: + FuncType() : args_(), results_() { initImmediateTypeId(); } + + FuncType(ValTypeVector&& args, ValTypeVector&& results) + : args_(std::move(args)), results_(std::move(results)) { + initImmediateTypeId(); + } + + FuncType(FuncType&&) = default; + FuncType& operator=(FuncType&&) = default; + + [[nodiscard]] bool clone(const FuncType& src) { + MOZ_ASSERT(args_.empty()); + MOZ_ASSERT(results_.empty()); + immediateTypeId_ = src.immediateTypeId_; + return args_.appendAll(src.args_) && results_.appendAll(src.results_); + } + + ValType arg(unsigned i) const { return args_[i]; } + const ValTypeVector& args() const { return args_; } + ValType result(unsigned i) const { return results_[i]; } + const ValTypeVector& results() const { return results_; } + + bool hasImmediateTypeId() const { + return immediateTypeId_ != NO_IMMEDIATE_TYPE_ID; + } + uint32_t immediateTypeId() const { + MOZ_ASSERT(hasImmediateTypeId()); + return immediateTypeId_; + } + + // The lsb for every immediate type id is set to distinguish an immediate type + // id from a type id represented by a pointer to the global hash type set. + static const uint32_t ImmediateBit = 0x1; + + HashNumber hash(const RecGroup* recGroup) const { + HashNumber hn = 0; + for (const ValType& vt : args_) { + hn = mozilla::AddToHash(hn, vt.forMatch(recGroup).hash()); + } + for (const ValType& vt : results_) { + hn = mozilla::AddToHash(hn, vt.forMatch(recGroup).hash()); + } + return hn; + } + + // Matches two function types for isorecursive equality. See + // "Matching type definitions" in WasmValType.h for more background. + static bool matches(const RecGroup* lhsRecGroup, const FuncType& lhs, + const RecGroup* rhsRecGroup, const FuncType& rhs) { + if (lhs.args_.length() != rhs.args_.length() || + lhs.results_.length() != rhs.results_.length()) { + return false; + } + for (uint32_t i = 0; i < lhs.args_.length(); i++) { + if (lhs.args_[i].forMatch(lhsRecGroup) != + rhs.args_[i].forMatch(rhsRecGroup)) { + return false; + } + } + for (uint32_t i = 0; i < lhs.results_.length(); i++) { + if (lhs.results_[i].forMatch(lhsRecGroup) != + rhs.results_[i].forMatch(rhsRecGroup)) { + return false; + } + } + return true; + } + + // Checks if every arg and result of the specified function types are bitwise + // equal. Type references must therefore point to exactly the same type + // definition instance. + static bool strictlyEquals(const FuncType& lhs, const FuncType& rhs) { + return EqualContainers(lhs.args(), rhs.args()) && + EqualContainers(lhs.results(), rhs.results()); + } + + // Checks if two function types are compatible in a given subtyping + // relationship. + static bool canBeSubTypeOf(const FuncType& subType, + const FuncType& superType) { + // A subtype must have exactly as many arguments as its supertype + if (subType.args().length() != superType.args().length()) { + return false; + } + + // A subtype must have exactly as many returns as its supertype + if (subType.results().length() != superType.results().length()) { + return false; + } + + // Function result types are covariant + for (uint32_t i = 0; i < superType.results().length(); i++) { + if (!ValType::isSubTypeOf(subType.results()[i], superType.results()[i])) { + return false; + } + } + + // Function argument types are contravariant + for (uint32_t i = 0; i < superType.args().length(); i++) { + if (!ValType::isSubTypeOf(superType.args()[i], subType.args()[i])) { + return false; + } + } + + return true; + } + + bool canHaveJitEntry() const; + bool canHaveJitExit() const; + + bool hasInt64Arg() const { + for (ValType arg : args()) { + if (arg.kind() == ValType::Kind::I64) { + return true; + } + } + return false; + } + + bool hasUnexposableArgOrRet() const { + for (ValType arg : args()) { + if (!arg.isExposable()) { + return true; + } + } + for (ValType result : results()) { + if (!result.isExposable()) { + return true; + } + } + return false; + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(FuncType); +}; + +//========================================================================= +// Structure types + +// The Module owns a dense array of StructType values that represent the +// structure types that the module knows about. It is created from the sparse +// array of types in the ModuleEnvironment when the Module is created. + +struct StructField { + FieldType type; + uint32_t offset; + bool isMutable; + + HashNumber hash(const RecGroup* recGroup) const { + HashNumber hn = 0; + hn = mozilla::AddToHash(hn, type.forMatch(recGroup).hash()); + hn = mozilla::AddToHash(hn, HashNumber(isMutable)); + return hn; + } + + // Checks if two struct fields are compatible in a given subtyping + // relationship. + static bool canBeSubTypeOf(const StructField& subType, + const StructField& superType) { + // Mutable fields are invariant w.r.t. field types + if (subType.isMutable && superType.isMutable) { + return subType.type == superType.type; + } + + // Immutable fields are covariant w.r.t. field types + if (!subType.isMutable && !superType.isMutable) { + return FieldType::isSubTypeOf(subType.type, superType.type); + } + + return false; + } +}; + +using StructFieldVector = Vector; + +using InlineTraceOffsetVector = Vector; +using OutlineTraceOffsetVector = Vector; + +class StructType { + public: + StructFieldVector fields_; // Field type, offset, and mutability + uint32_t size_; // The size of the type in bytes. + InlineTraceOffsetVector inlineTraceOffsets_; + OutlineTraceOffsetVector outlineTraceOffsets_; + + public: + StructType() : fields_(), size_(0) {} + + explicit StructType(StructFieldVector&& fields) + : fields_(std::move(fields)), size_(0) {} + + StructType(StructType&&) = default; + StructType& operator=(StructType&&) = default; + + [[nodiscard]] bool clone(const StructType& src) { + if (!fields_.appendAll(src.fields_)) { + return false; + } + size_ = src.size_; + return true; + } + + [[nodiscard]] bool init(); + + bool isDefaultable() const { + for (auto& field : fields_) { + if (!field.type.isDefaultable()) { + return false; + } + } + return true; + } + + HashNumber hash(const RecGroup* recGroup) const { + HashNumber hn = 0; + for (const StructField& field : fields_) { + hn = mozilla::AddToHash(hn, field.hash(recGroup)); + } + return hn; + } + + // Matches two struct types for isorecursive equality. See + // "Matching type definitions" in WasmValType.h for more background. + static bool matches(const RecGroup* lhsRecGroup, const StructType& lhs, + const RecGroup* rhsRecGroup, const StructType& rhs) { + if (lhs.fields_.length() != rhs.fields_.length()) { + return false; + } + for (uint32_t i = 0; i < lhs.fields_.length(); i++) { + const StructField& lhsField = lhs.fields_[i]; + const StructField& rhsField = rhs.fields_[i]; + if (lhsField.isMutable != rhsField.isMutable || + lhsField.type.forMatch(lhsRecGroup) != + rhsField.type.forMatch(rhsRecGroup)) { + return false; + } + } + return true; + } + + // Checks if two struct types are compatible in a given subtyping + // relationship. + static bool canBeSubTypeOf(const StructType& subType, + const StructType& superType) { + // A subtype must have at least as many fields as its supertype + if (subType.fields_.length() < superType.fields_.length()) { + return false; + } + + // Every field that is in both superType and subType must be compatible + for (uint32_t i = 0; i < superType.fields_.length(); i++) { + if (!StructField::canBeSubTypeOf(subType.fields_[i], + superType.fields_[i])) { + return false; + } + } + return true; + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(StructType); +}; + +using StructTypeVector = Vector; + +// Utility for computing field offset and alignments, and total size for +// structs and tags. This is complicated by fact that a WasmStructObject has +// an inline area, which is used first, and if that fills up an optional +// C++-heap-allocated outline area is used. We need to be careful not to +// split any data item across the boundary. This is ensured as follows: +// +// (1) the possible field sizes are 1, 2, 4, 8 and 16 only. +// (2) each field is "naturally aligned" -- aligned to its size. +// (3) MaxInlineBytes (the size of the inline area) % 16 == 0. +// +// From (1) and (2), it follows that all fields are placed so that their first +// and last bytes fall within the same 16-byte chunk. That is, +// offset_of_first_byte_of_field / 16 == offset_of_last_byte_of_field / 16. +// +// Given that, it follows from (3) that all fields fall completely within +// either the inline or outline areas; no field crosses the boundary. +class StructLayout { + CheckedInt32 sizeSoFar = 0; + uint32_t structAlignment = 1; + + public: + // The field adders return the offset of the the field. + CheckedInt32 addField(FieldType type); + + // The close method rounds up the structure size to the appropriate + // alignment and returns that size. + CheckedInt32 close(); +}; + +//========================================================================= +// Array types + +class ArrayType { + public: + FieldType elementType_; // field type + bool isMutable_; // mutability + + public: + ArrayType() : isMutable_(false) {} + ArrayType(FieldType elementType, bool isMutable) + : elementType_(elementType), isMutable_(isMutable) {} + + ArrayType(const ArrayType&) = default; + ArrayType& operator=(const ArrayType&) = default; + + ArrayType(ArrayType&&) = default; + ArrayType& operator=(ArrayType&&) = default; + + [[nodiscard]] bool clone(const ArrayType& src) { + elementType_ = src.elementType_; + isMutable_ = src.isMutable_; + return true; + } + + bool isDefaultable() const { return elementType_.isDefaultable(); } + + HashNumber hash(const RecGroup* recGroup) const { + HashNumber hn = 0; + hn = mozilla::AddToHash(hn, elementType_.forMatch(recGroup).hash()); + hn = mozilla::AddToHash(hn, HashNumber(isMutable_)); + return hn; + } + + // Matches two array types for isorecursive equality. See + // "Matching type definitions" in WasmValType.h for more background. + static bool matches(const RecGroup* lhsRecGroup, const ArrayType& lhs, + const RecGroup* rhsRecGroup, const ArrayType& rhs) { + if (lhs.isMutable_ != rhs.isMutable_ || + lhs.elementType_.forMatch(lhsRecGroup) != + rhs.elementType_.forMatch(rhsRecGroup)) { + return false; + } + return true; + } + + // Checks if two arrays are compatible in a given subtyping relationship. + static bool canBeSubTypeOf(const ArrayType& subType, + const ArrayType& superType) { + // Mutable fields are invariant w.r.t. field types + if (subType.isMutable_ && superType.isMutable_) { + return subType.elementType_ == superType.elementType_; + } + + // Immutable fields are covariant w.r.t. field types + if (!subType.isMutable_ && !superType.isMutable_) { + return FieldType::isSubTypeOf(subType.elementType_, + superType.elementType_); + } + + return true; + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +WASM_DECLARE_CACHEABLE_POD(ArrayType); + +using ArrayTypeVector = Vector; + +//========================================================================= +// SuperTypeVector + +// [SMDOC] Super type vector +// +// A super type vector is a vector representation of the linked list of super +// types that a type definition has. Every element is a raw pointer to a type +// definition. It's possible to form a vector here because type definitions +// are trees, not DAGs, with every type having at most one super type. +// +// The first element in the vector is the 'root' type definition without a +// super type. The last element is to the type definition itself. +// +// ## Subtype checking +// +// The only purpose of a super type vector is to support constant time +// subtyping checks. This is not free, it comes at the cost of worst case N^2 +// metadata size growth. We limit the max subtyping depth to counter this. +// +// To perform a subtype check we rely on the following: +// (1) a type A is a subtype (<:) of type B iff: +// type A == type B OR +// type B is reachable by following declared super types of type A +// (2) we order super type vectors from least to most derived types +// (3) the 'subtyping depth' of all type definitions is statically known +// +// With the above, we know that if type B is a super type of type A, that it +// must be in A's super type vector at type B's subtyping depth. We can +// therefore just do an index and comparison to determine if that's the case. +// +// ## Example +// +// For the following type section: +// .. +// 12: (type (struct)) +// .. +// 34: (type (sub 12 (struct))) +// .. +// 56: (type (sub 34 (struct))) +// .. +// 78: (type (sub 56 (struct))) +// .. +// +// (type 12) would have the following super type vector: +// [(type 12)] +// +// (type 78) would have the following super type vector: +// [(type 12), (type 34), (type 56), (type 78)] +// +// Checking that (type 78) <: (type 12) can use the fact that (type 12) will +// always be present at depth 0 of any super type vector it is in, and +// therefore check the vector at that index. +// +// ## Minimum sizing +// +// As a further optimization to avoid bounds checking, we guarantee that all +// super type vectors are at least `MinSuperTypeVectorLength`. All checks +// against indices that we know statically are at/below that can skip bounds +// checking. Extra entries added to reach the minimum size are initialized to +// null. +class SuperTypeVector { + SuperTypeVector() : typeDef_(nullptr), length_(0) {} + + // The TypeDef for which this is the supertype vector. That TypeDef should + // point back to this SuperTypeVector. + const TypeDef* typeDef_; + + // The length of types stored inline below. + uint32_t length_; + + public: + // Raw pointers to the super types of this type definition. Ordered from + // least-derived to most-derived. Do not add any fields after this point. + const SuperTypeVector* types_[0]; + + // Batch allocate super type vectors for all the types in a recursion group. + // Returns a pointer to the first super type vector, which can be used to + // free all vectors. + [[nodiscard]] static const SuperTypeVector* createMultipleForRecGroup( + RecGroup* recGroup); + + const TypeDef* typeDef() const { return typeDef_; } + void setTypeDef(const TypeDef* typeDef) { typeDef_ = typeDef; } + + uint32_t length() const { return length_; } + void setLength(uint32_t length) { length_ = length; } + + const SuperTypeVector* type(size_t index) const { + MOZ_ASSERT(index < length_); + return types_[index]; + } + void setType(size_t index, const SuperTypeVector* type) { + MOZ_ASSERT(index < length_); + types_[index] = type; + } + + // The length of a super type vector for a specific type def. + static size_t lengthForTypeDef(const TypeDef& typeDef); + // The byte size of a super type vector for a specific type def. + static size_t byteSizeForTypeDef(const TypeDef& typeDef); + + static size_t offsetOfLength() { return offsetof(SuperTypeVector, length_); } + static size_t offsetOfSelfTypeDef() { + return offsetof(SuperTypeVector, typeDef_); + }; + static size_t offsetOfTypeDefInVector(uint32_t typeDefDepth); +}; + +// Ensure it is safe to use `sizeof(SuperTypeVector)` to find the offset of +// `types_[0]`. +static_assert(offsetof(SuperTypeVector, types_) == sizeof(SuperTypeVector)); + +//========================================================================= +// TypeDef and supporting types + +// A tagged container for the various types that can be present in a wasm +// module's type section. + +enum class TypeDefKind : uint8_t { + None = 0, + Func, + Struct, + Array, +}; + +class TypeDef { + uint32_t offsetToRecGroup_; + + // The supertype vector for this TypeDef. That SuperTypeVector should point + // back to this TypeDef. + const SuperTypeVector* superTypeVector_; + + const TypeDef* superTypeDef_; + uint16_t subTypingDepth_; + TypeDefKind kind_; + union { + FuncType funcType_; + StructType structType_; + ArrayType arrayType_; + }; + + void setRecGroup(RecGroup* recGroup) { + uintptr_t recGroupAddr = (uintptr_t)recGroup; + uintptr_t typeDefAddr = (uintptr_t)this; + MOZ_ASSERT(typeDefAddr > recGroupAddr); + MOZ_ASSERT(typeDefAddr - recGroupAddr <= UINT32_MAX); + offsetToRecGroup_ = typeDefAddr - recGroupAddr; + } + + public: + explicit TypeDef(RecGroup* recGroup) + : offsetToRecGroup_(0), + superTypeVector_(nullptr), + superTypeDef_(nullptr), + subTypingDepth_(0), + kind_(TypeDefKind::None) { + setRecGroup(recGroup); + } + + ~TypeDef() { + switch (kind_) { + case TypeDefKind::Func: + funcType_.~FuncType(); + break; + case TypeDefKind::Struct: + structType_.~StructType(); + break; + case TypeDefKind::Array: + arrayType_.~ArrayType(); + break; + case TypeDefKind::None: + break; + } + } + + TypeDef& operator=(FuncType&& that) noexcept { + MOZ_ASSERT(isNone()); + kind_ = TypeDefKind::Func; + new (&funcType_) FuncType(std::move(that)); + return *this; + } + + TypeDef& operator=(StructType&& that) noexcept { + MOZ_ASSERT(isNone()); + kind_ = TypeDefKind::Struct; + new (&structType_) StructType(std::move(that)); + return *this; + } + + TypeDef& operator=(ArrayType&& that) noexcept { + MOZ_ASSERT(isNone()); + kind_ = TypeDefKind::Array; + new (&arrayType_) ArrayType(std::move(that)); + return *this; + } + + const SuperTypeVector* superTypeVector() const { return superTypeVector_; } + + void setSuperTypeVector(const SuperTypeVector* superTypeVector) { + superTypeVector_ = superTypeVector; + } + + static size_t offsetOfKind() { return offsetof(TypeDef, kind_); } + + static size_t offsetOfSuperTypeVector() { + return offsetof(TypeDef, superTypeVector_); + } + + const TypeDef* superTypeDef() const { return superTypeDef_; } + + uint16_t subTypingDepth() const { return subTypingDepth_; } + + const RecGroup& recGroup() const { + uintptr_t typeDefAddr = (uintptr_t)this; + uintptr_t recGroupAddr = typeDefAddr - offsetToRecGroup_; + return *(const RecGroup*)recGroupAddr; + } + + TypeDefKind kind() const { return kind_; } + + bool isNone() const { return kind_ == TypeDefKind::None; } + + bool isFuncType() const { return kind_ == TypeDefKind::Func; } + + bool isStructType() const { return kind_ == TypeDefKind::Struct; } + + bool isArrayType() const { return kind_ == TypeDefKind::Array; } + + const FuncType& funcType() const { + MOZ_ASSERT(isFuncType()); + return funcType_; + } + + FuncType& funcType() { + MOZ_ASSERT(isFuncType()); + return funcType_; + } + + const StructType& structType() const { + MOZ_ASSERT(isStructType()); + return structType_; + } + + StructType& structType() { + MOZ_ASSERT(isStructType()); + return structType_; + } + + const ArrayType& arrayType() const { + MOZ_ASSERT(isArrayType()); + return arrayType_; + } + + ArrayType& arrayType() { + MOZ_ASSERT(isArrayType()); + return arrayType_; + } + + // Get a value that can be used for matching type definitions across + // different recursion groups. + static inline uintptr_t forMatch(const TypeDef* typeDef, + const RecGroup* recGroup); + + HashNumber hash() const { + HashNumber hn = HashNumber(kind_); + hn = mozilla::AddToHash(hn, TypeDef::forMatch(superTypeDef_, &recGroup())); + switch (kind_) { + case TypeDefKind::Func: + hn = mozilla::AddToHash(hn, funcType_.hash(&recGroup())); + break; + case TypeDefKind::Struct: + hn = mozilla::AddToHash(hn, structType_.hash(&recGroup())); + break; + case TypeDefKind::Array: + hn = mozilla::AddToHash(hn, arrayType_.hash(&recGroup())); + break; + case TypeDefKind::None: + break; + } + return hn; + } + + // Matches two type definitions for isorecursive equality. See + // "Matching type definitions" in WasmValType.h for more background. + static bool matches(const TypeDef& lhs, const TypeDef& rhs) { + if (lhs.kind_ != rhs.kind_) { + return false; + } + if (TypeDef::forMatch(lhs.superTypeDef_, &lhs.recGroup()) != + TypeDef::forMatch(rhs.superTypeDef_, &rhs.recGroup())) { + return false; + } + switch (lhs.kind_) { + case TypeDefKind::Func: + return FuncType::matches(&lhs.recGroup(), lhs.funcType_, + &rhs.recGroup(), rhs.funcType_); + case TypeDefKind::Struct: + return StructType::matches(&lhs.recGroup(), lhs.structType_, + &rhs.recGroup(), rhs.structType_); + case TypeDefKind::Array: + return ArrayType::matches(&lhs.recGroup(), lhs.arrayType_, + &rhs.recGroup(), rhs.arrayType_); + case TypeDefKind::None: + return true; + } + return false; + } + + // Checks if two type definitions are compatible in a given subtyping + // relationship. + static bool canBeSubTypeOf(const TypeDef* subType, const TypeDef* superType) { + if (subType->kind() != superType->kind()) { + return false; + } + + switch (subType->kind_) { + case TypeDefKind::Func: + return FuncType::canBeSubTypeOf(subType->funcType_, + superType->funcType_); + case TypeDefKind::Struct: + return StructType::canBeSubTypeOf(subType->structType_, + superType->structType_); + case TypeDefKind::Array: + return ArrayType::canBeSubTypeOf(subType->arrayType_, + superType->arrayType_); + case TypeDefKind::None: + MOZ_CRASH(); + } + return false; + } + + void setSuperTypeDef(const TypeDef* superTypeDef) { + superTypeDef_ = superTypeDef; + subTypingDepth_ = superTypeDef_->subTypingDepth_ + 1; + } + + // Checks if `subTypeDef` is a declared sub type of `superTypeDef`. + static bool isSubTypeOf(const TypeDef* subTypeDef, + const TypeDef* superTypeDef) { + // Fast path for when the types are equal + if (MOZ_LIKELY(subTypeDef == superTypeDef)) { + return true; + } + const SuperTypeVector* subSuperTypeVector = subTypeDef->superTypeVector(); + + // During construction of a recursion group, the super type vector may not + // have been computed yet, in which case we need to fall back to a linear + // search. + if (!subSuperTypeVector) { + while (subTypeDef) { + if (subTypeDef == superTypeDef) { + return true; + } + subTypeDef = subTypeDef->superTypeDef(); + } + return false; + } + + // The supertype vector does exist. So check it points back here. + MOZ_ASSERT(subSuperTypeVector->typeDef() == subTypeDef); + + // We need to check if `superTypeDef` is one of `subTypeDef`s super types + // by checking in `subTypeDef`s super type vector. We can use the static + // information of the depth of `superTypeDef` to index directly into the + // vector. + uint32_t subTypingDepth = superTypeDef->subTypingDepth(); + if (subTypingDepth >= subSuperTypeVector->length()) { + return false; + } + + const SuperTypeVector* superSuperTypeVector = + superTypeDef->superTypeVector(); + MOZ_ASSERT(superSuperTypeVector); + MOZ_ASSERT(superSuperTypeVector->typeDef() == superTypeDef); + + return subSuperTypeVector->type(subTypingDepth) == superSuperTypeVector; + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + WASM_DECLARE_FRIEND_SERIALIZE(TypeDef); +}; + +using SharedTypeDef = RefPtr; +using MutableTypeDef = RefPtr; + +using TypeDefVector = Vector; +using TypeDefPtrVector = Vector; + +using TypeDefPtrToIndexMap = + HashMap, + SystemAllocPolicy>; + +//========================================================================= +// RecGroup + +// A recursion group is a set of type definitions that may refer to each other +// or to type definitions in another recursion group. There is an ordering +// restriction on type references such that references across recursion groups +// must be acyclic. +// +// Type definitions are stored inline in their containing recursion group, and +// have an offset to their containing recursion group. Recursion groups are +// atomically refcounted and hold strong references to other recursion groups +// they depend on. +// +// Type equality is structural in WebAssembly, and we canonicalize recursion +// groups while building them so that pointer equality of types implies +// equality of types. There is a global hash set of weak pointers to recursion +// groups that holds the current canonical instance of a recursion group. +class RecGroup : public AtomicRefCounted { + // Whether this recursion group has been finished and acquired strong + // references to external recursion groups. + bool finalizedTypes_; + // The number of types stored in this recursion group. + uint32_t numTypes_; + // The batch allocated super type vectors for all type definitions in this + // recursion group. + const SuperTypeVector* vectors_; + // The first type definition stored inline in this recursion group. + TypeDef types_[0]; + + friend class TypeContext; + + explicit RecGroup(uint32_t numTypes) + : finalizedTypes_(false), numTypes_(numTypes), vectors_(nullptr) {} + + // Compute the size in bytes of a recursion group with the specified amount + // of types. + static constexpr size_t sizeOfRecGroup(uint32_t numTypes) { + static_assert(MaxTypes <= SIZE_MAX / sizeof(TypeDef)); + return sizeof(RecGroup) + sizeof(TypeDef) * numTypes; + } + + // Allocate a recursion group with the specified amount of types. The type + // definitions will be ready to be filled in. Users must call `finish` once + // type definitions are initialized so that strong references to external + // recursion groups are taken. + static RefPtr allocate(uint32_t numTypes) { + // Allocate the recursion group with the correct size + RecGroup* recGroup = (RecGroup*)js_malloc(sizeOfRecGroup(numTypes)); + if (!recGroup) { + return nullptr; + } + + // Construct the recursion group and types that are stored inline + new (recGroup) RecGroup(numTypes); + for (uint32_t i = 0; i < numTypes; i++) { + new (recGroup->types_ + i) TypeDef(recGroup); + } + return recGroup; + } + + // Finish initialization by acquiring strong references to groups referenced + // by type definitions. + [[nodiscard]] bool finalizeDefinitions() { + MOZ_ASSERT(!finalizedTypes_); + // Super type vectors are only needed for GC and have a size/time impact + // that we don't want to encur until we're ready for it. Only use them when + // GC is built into the binary. +#ifdef ENABLE_WASM_GC + vectors_ = SuperTypeVector::createMultipleForRecGroup(this); + if (!vectors_) { + return false; + } +#endif + visitReferencedGroups([](const RecGroup* recGroup) { recGroup->AddRef(); }); + finalizedTypes_ = true; + return true; + } + + // Visit every external recursion group that is referenced by the types in + // this recursion group. + template + void visitReferencedGroups(Visitor visitor) const { + auto visitValType = [this, visitor](ValType type) { + if (type.isTypeRef() && &type.typeDef()->recGroup() != this) { + visitor(&type.typeDef()->recGroup()); + } + }; + auto visitFieldType = [this, visitor](FieldType type) { + if (type.isTypeRef() && &type.typeDef()->recGroup() != this) { + visitor(&type.typeDef()->recGroup()); + } + }; + + for (uint32_t i = 0; i < numTypes_; i++) { + const TypeDef& typeDef = types_[i]; + + if (typeDef.superTypeDef() && + &typeDef.superTypeDef()->recGroup() != this) { + visitor(&typeDef.superTypeDef()->recGroup()); + } + + switch (typeDef.kind()) { + case TypeDefKind::Func: { + const FuncType& funcType = typeDef.funcType(); + for (auto type : funcType.args()) { + visitValType(type); + } + for (auto type : funcType.results()) { + visitValType(type); + } + break; + } + case TypeDefKind::Struct: { + const StructType& structType = typeDef.structType(); + for (const auto& field : structType.fields_) { + visitFieldType(field.type); + } + break; + } + case TypeDefKind::Array: { + const ArrayType& arrayType = typeDef.arrayType(); + visitFieldType(arrayType.elementType_); + break; + } + case TypeDefKind::None: { + MOZ_CRASH(); + } + } + } + } + + public: + ~RecGroup() { + // Release the referenced recursion groups if we acquired references to + // them. Do this before the type definitions are destroyed below. + if (finalizedTypes_) { + finalizedTypes_ = false; + visitReferencedGroups( + [](const RecGroup* recGroup) { recGroup->Release(); }); + } + + if (vectors_) { + js_free((void*)vectors_); + vectors_ = nullptr; + } + + // Call destructors on all the type definitions. + for (uint32_t i = 0; i < numTypes_; i++) { + type(i).~TypeDef(); + } + } + + // Recursion groups cannot be copied or moved + RecGroup& operator=(const RecGroup&) = delete; + RecGroup& operator=(RecGroup&&) = delete; + + // Get the type definition at the group type index (not module type index). + TypeDef& type(uint32_t groupTypeIndex) { + // We cannot mutate type definitions after we've finalized them + MOZ_ASSERT(!finalizedTypes_); + return types_[groupTypeIndex]; + } + const TypeDef& type(uint32_t groupTypeIndex) const { + return types_[groupTypeIndex]; + } + + // The number of types stored in this recursion group. + uint32_t numTypes() const { return numTypes_; } + + // Get the index of a type definition that's in this recursion group. + uint32_t indexOf(const TypeDef* typeDef) const { + MOZ_ASSERT(typeDef >= types_); + size_t groupTypeIndex = (size_t)(typeDef - types_); + MOZ_ASSERT(groupTypeIndex < numTypes()); + return (uint32_t)groupTypeIndex; + } + + HashNumber hash() const { + HashNumber hn = 0; + for (uint32_t i = 0; i < numTypes(); i++) { + hn = mozilla::AddToHash(hn, types_[i].hash()); + } + return hn; + } + + // Matches two recursion groups for isorecursive equality. See + // "Matching type definitions" in WasmValType.h for more background. + static bool matches(const RecGroup& lhs, const RecGroup& rhs) { + if (lhs.numTypes() != rhs.numTypes()) { + return false; + } + for (uint32_t i = 0; i < lhs.numTypes(); i++) { + if (!TypeDef::matches(lhs.type(i), rhs.type(i))) { + return false; + } + } + return true; + } +}; + +// Remove all types from the canonical type set that are not referenced from +// outside the type set. +extern void PurgeCanonicalTypes(); + +using SharedRecGroup = RefPtr; +using MutableRecGroup = RefPtr; +using SharedRecGroupVector = Vector; + +//========================================================================= +// TypeContext + +// A type context holds the recursion groups and corresponding type definitions +// defined in a module. +class TypeContext : public AtomicRefCounted { + FeatureArgs features_; + // The pending recursion group that is currently being constructed + MutableRecGroup pendingRecGroup_; + // An in-order list of all the recursion groups defined in this module + SharedRecGroupVector recGroups_; + // An in-order list of the type definitions in the module. Each type is + // stored in a recursion group. + TypeDefPtrVector types_; + // A map from type definition to the original module index. + TypeDefPtrToIndexMap moduleIndices_; + + static SharedRecGroup canonicalizeGroup(SharedRecGroup recGroup); + + public: + TypeContext() = default; + explicit TypeContext(const FeatureArgs& features) : features_(features) {} + ~TypeContext(); + + size_t sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const { + return types_.sizeOfExcludingThis(mallocSizeOf) + + moduleIndices_.shallowSizeOfExcludingThis(mallocSizeOf); + } + + // Disallow copy, allow move initialization + TypeContext(const TypeContext&) = delete; + TypeContext& operator=(const TypeContext&) = delete; + TypeContext(TypeContext&&) = delete; + TypeContext& operator=(TypeContext&&) = delete; + + // Begin creating a recursion group with the specified number of types. + // Returns a recursion group to be filled in with type definitions. This must + // be paired with `endGroup`. + [[nodiscard]] MutableRecGroup startRecGroup(uint32_t numTypes) { + // We must not have a pending group + MOZ_ASSERT(!pendingRecGroup_); + + // Create the group and add it to the list of groups + pendingRecGroup_ = RecGroup::allocate(numTypes); + if (!pendingRecGroup_ || !recGroups_.append(pendingRecGroup_)) { + return nullptr; + } + + // Store the types of the group into our index space maps. These may get + // overwritten when we finish this group and canonicalize it. We need to do + // this before finishing, because these entries will be used by decoding + // and error printing. + for (uint32_t groupTypeIndex = 0; groupTypeIndex < numTypes; + groupTypeIndex++) { + const TypeDef* typeDef = &pendingRecGroup_->type(groupTypeIndex); + uint32_t typeIndex = types_.length(); + if (!types_.append(typeDef) || !moduleIndices_.put(typeDef, typeIndex)) { + return nullptr; + } + } + return pendingRecGroup_; + } + + // Finish creation of a recursion group after type definitions have been + // initialized. This must be paired with `startGroup`. + [[nodiscard]] bool endRecGroup() { + // We must have started a recursion group + MOZ_ASSERT(pendingRecGroup_); + MutableRecGroup recGroup = pendingRecGroup_; + pendingRecGroup_ = nullptr; + + // Finalize the type definitions in the recursion group + if (!recGroup->finalizeDefinitions()) { + return false; + } + + // Canonicalize the recursion group + SharedRecGroup canonicalRecGroup = canonicalizeGroup(recGroup); + if (!canonicalRecGroup) { + return false; + } + + // Nothing left to do if this group became the canonical group + if (canonicalRecGroup == recGroup) { + return true; + } + + // Store the canonical group into the list + recGroups_.back() = canonicalRecGroup; + + // Overwrite all the entries we stored into the index space maps when we + // started this group. + MOZ_ASSERT(recGroup->numTypes() == canonicalRecGroup->numTypes()); + for (uint32_t groupTypeIndex = 0; groupTypeIndex < recGroup->numTypes(); + groupTypeIndex++) { + uint32_t typeIndex = length() - recGroup->numTypes() + groupTypeIndex; + const TypeDef* oldTypeDef = types_[typeIndex]; + const TypeDef* newTypeDef = &canonicalRecGroup->type(groupTypeIndex); + types_[typeIndex] = newTypeDef; + moduleIndices_.remove(oldTypeDef); + if (!moduleIndices_.put(newTypeDef, typeIndex)) { + return false; + } + } + + return true; + } + + template + [[nodiscard]] bool addType(T&& type) { + MutableRecGroup recGroup = startRecGroup(1); + if (!recGroup) { + return false; + } + recGroup->type(0) = std::move(type); + return endRecGroup(); + } + + const TypeDef& type(uint32_t index) const { return *types_[index]; } + const TypeDef& operator[](uint32_t index) const { return *types_[index]; } + + bool empty() const { return types_.empty(); } + uint32_t length() const { return types_.length(); } + + const SharedRecGroupVector& groups() const { return recGroups_; } + + // Map from type definition to index + + uint32_t indexOf(const TypeDef& typeDef) const { + auto moduleIndex = moduleIndices_.readonlyThreadsafeLookup(&typeDef); + MOZ_RELEASE_ASSERT(moduleIndex.found()); + return moduleIndex->value(); + } +}; + +using SharedTypeContext = RefPtr; +using MutableTypeContext = RefPtr; + +//========================================================================= +// TypeHandle + +// An unambiguous strong reference to a type definition in a specific type +// context. +class TypeHandle { + private: + SharedTypeContext context_; + uint32_t index_; + + public: + TypeHandle(SharedTypeContext context, uint32_t index) + : context_(context), index_(index) { + MOZ_ASSERT(index_ < context_->length()); + } + TypeHandle(SharedTypeContext context, const TypeDef& def) + : context_(context), index_(context->indexOf(def)) {} + + TypeHandle(const TypeHandle&) = default; + TypeHandle& operator=(const TypeHandle&) = default; + + const SharedTypeContext& context() const { return context_; } + uint32_t index() const { return index_; } + const TypeDef& def() const { return context_->type(index_); } +}; + +//========================================================================= +// misc + +/* static */ +inline uintptr_t TypeDef::forMatch(const TypeDef* typeDef, + const RecGroup* recGroup) { + // TypeDef is aligned sufficiently to allow a tag to distinguish a local type + // reference (index) from a non-local type reference (pointer). + static_assert(alignof(TypeDef) > 1); + MOZ_ASSERT((uintptr_t(typeDef) & 0x1) == 0); + + // Return a tagged index for local type references + if (typeDef && &typeDef->recGroup() == recGroup) { + return uintptr_t(recGroup->indexOf(typeDef)) | 0x1; + } + + // Return an untagged pointer for non-local type references + return uintptr_t(typeDef); +} + +/* static */ +inline MatchTypeCode MatchTypeCode::forMatch(PackedTypeCode ptc, + const RecGroup* recGroup) { + MatchTypeCode mtc = {}; + mtc.typeCode = PackedRepr(ptc.typeCode()); + mtc.typeRef = TypeDef::forMatch(ptc.typeDef(), recGroup); + mtc.nullable = ptc.isNullable(); + return mtc; +} + +inline RefTypeHierarchy RefType::hierarchy() const { + switch (kind()) { + case RefType::Func: + case RefType::NoFunc: + return RefTypeHierarchy::Func; + case RefType::Extern: + case RefType::NoExtern: + return RefTypeHierarchy::Extern; + case RefType::Any: + case RefType::None: + case RefType::Eq: + case RefType::Struct: + case RefType::Array: + return RefTypeHierarchy::Any; + case RefType::TypeRef: + switch (typeDef()->kind()) { + case TypeDefKind::Struct: + case TypeDefKind::Array: + return RefTypeHierarchy::Any; + case TypeDefKind::Func: + return RefTypeHierarchy::Func; + case TypeDefKind::None: + MOZ_CRASH(); + } + } + MOZ_CRASH("switch is exhaustive"); +} + +inline TableRepr RefType::tableRepr() const { + switch (hierarchy()) { + case RefTypeHierarchy::Any: + case RefTypeHierarchy::Extern: + return TableRepr::Ref; + case RefTypeHierarchy::Func: + return TableRepr::Func; + } + MOZ_CRASH("switch is exhaustive"); +} + +inline bool RefType::isFuncHierarchy() const { + return hierarchy() == RefTypeHierarchy::Func; +} +inline bool RefType::isExternHierarchy() const { + return hierarchy() == RefTypeHierarchy::Extern; +} +inline bool RefType::isAnyHierarchy() const { + return hierarchy() == RefTypeHierarchy::Any; +} + +/* static */ +inline bool RefType::isSubTypeOf(RefType subType, RefType superType) { + // Anything is a subtype of itself. + if (subType == superType) { + return true; + } + + // A subtype must have the same nullability as the supertype or the + // supertype must be nullable. + if (subType.isNullable() && !superType.isNullable()) { + return false; + } + + // Non type-index references are subtypes if they have the same kind + if (!subType.isTypeRef() && !superType.isTypeRef() && + subType.kind() == superType.kind()) { + return true; + } + + // eqref is a subtype of anyref + if (subType.isEq() && superType.isAny()) { + return true; + } + + // structref/arrayref are subtypes of eqref and anyref + if ((subType.isStruct() || subType.isArray()) && + (superType.isAny() || superType.isEq())) { + return true; + } + + // Structs are subtypes of structref, eqref and anyref + if (subType.isTypeRef() && subType.typeDef()->isStructType() && + (superType.isAny() || superType.isEq() || superType.isStruct())) { + return true; + } + + // Arrays are subtypes of arrayref, eqref and anyref + if (subType.isTypeRef() && subType.typeDef()->isArrayType() && + (superType.isAny() || superType.isEq() || superType.isArray())) { + return true; + } + + // Funcs are subtypes of funcref + if (subType.isTypeRef() && subType.typeDef()->isFuncType() && + superType.isFunc()) { + return true; + } + + // Type references can be subtypes + if (subType.isTypeRef() && superType.isTypeRef()) { + return TypeDef::isSubTypeOf(subType.typeDef(), superType.typeDef()); + } + + // No func is the bottom type of the func hierarchy + if (subType.isNoFunc() && superType.hierarchy() == RefTypeHierarchy::Func) { + return true; + } + + // No extern is the bottom type of the extern hierarchy + if (subType.isNoExtern() && + superType.hierarchy() == RefTypeHierarchy::Extern) { + return true; + } + + // None is the bottom type of the any hierarchy + if (subType.isNone() && superType.hierarchy() == RefTypeHierarchy::Any) { + return true; + } + + return false; +} + +/* static */ +inline bool RefType::castPossible(RefType sourceType, RefType destType) { + // Nullable types always have null in common. + if (sourceType.isNullable() && destType.isNullable()) { + return true; + } + + // At least one of the types is non-nullable, so the only common values can be + // non-null. Therefore, if either type is a bottom type, common values are + // impossible. + if (sourceType.isRefBottom() || destType.isRefBottom()) { + return false; + } + + // After excluding bottom types, our type hierarchy is a tree, and after + // excluding nulls, subtype relationships are sufficient to tell if the types + // share any values. If neither type is a subtype of the other, then they are + // on different branches of the tree and completely disjoint. + RefType sourceNonNull = sourceType.withIsNullable(false); + RefType destNonNull = destType.withIsNullable(false); + return RefType::isSubTypeOf(sourceNonNull, destNonNull) || + RefType::isSubTypeOf(destNonNull, sourceNonNull); +} + +//========================================================================= +// [SMDOC] Signatures and runtime types +// +// TypeIdDesc describes the runtime representation of a TypeDef suitable for +// type equality checks. The kind of representation depends on whether the type +// is a function or a GC type. This design is in flux and will evolve. +// +// # Function types +// +// For functions in the general case, a FuncType is allocated and stored in a +// process-wide hash table, so that pointer equality implies structural +// equality. This process does not correctly handle type references (which would +// require hash-consing of infinite-trees), but that's okay while +// function-references and gc-types are experimental. +// +// A pointer to the hash table entry is stored in the global data +// area for each instance, and TypeIdDesc gives the offset to this entry. +// +// ## Immediate function types +// +// As an optimization for the 99% case where the FuncType has a small number of +// parameters, the FuncType is bit-packed into a uint32 immediate value so that +// integer equality implies structural equality. Both cases can be handled with +// a single comparison by always setting the LSB for the immediates +// (the LSB is necessarily 0 for allocated FuncType pointers due to alignment). +// +// # GC types +// +// For GC types, an entry is always created in the global data area and a +// unique RttValue (see wasm/TypedObject.h) is stored there. This RttValue +// is the value given by 'rtt.canon $t' for each type definition. As each entry +// is given a unique value and no canonicalization is done (which would require +// hash-consing of infinite-trees), this is not yet spec compliant. +// +// # wasm::Instance and the global type context +// +// As GC objects (aka TypedObject) may outlive the module they are created in, +// types are additionally transferred to a wasm::Context (which is part of +// JSContext) upon instantiation. This wasm::Context contains the +// 'global type context' that RTTValues refer to by type index. Types are never +// freed from the global type context as that would shift the index space. In +// the future, this will be fixed. + +} // namespace wasm +} // namespace js + +#endif // wasm_type_def_h diff --git a/js/src/wasm/WasmUtility.h b/js/src/wasm/WasmUtility.h new file mode 100644 index 0000000000..fc9f0e54b7 --- /dev/null +++ b/js/src/wasm/WasmUtility.h @@ -0,0 +1,65 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef wasm_utility_h +#define wasm_utility_h + +#include +namespace js { +namespace wasm { + +template +static inline bool EqualContainers(const Container1& lhs, + const Container2& rhs) { + return std::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); +} + +#define WASM_DECLARE_POD_VECTOR(Type, VectorName) \ + } \ + } \ + static_assert(std::is_trivially_copyable::value, \ + "Must be trivially copyable"); \ + static_assert(std::is_trivially_destructible::value, \ + "Must be trivially destructible"); \ + namespace js { \ + namespace wasm { \ + typedef Vector VectorName; + +using mozilla::MallocSizeOf; + +template +static inline size_t SizeOfVectorElementExcludingThis( + const T& elem, MallocSizeOf mallocSizeOf) { + return elem.sizeOfExcludingThis(mallocSizeOf); +} + +template +static inline size_t SizeOfVectorElementExcludingThis( + const RefPtr& elem, MallocSizeOf mallocSizeOf) { + return elem->sizeOfExcludingThis(mallocSizeOf); +} + +template +static inline size_t SizeOfVectorExcludingThis( + const mozilla::Vector& vec, + MallocSizeOf mallocSizeOf) { + size_t size = vec.sizeOfExcludingThis(mallocSizeOf); + for (const T& t : vec) { + size += SizeOfVectorElementExcludingThis(t, mallocSizeOf); + } + return size; +} + +template +static inline size_t SizeOfMaybeExcludingThis(const mozilla::Maybe& maybe, + MallocSizeOf mallocSizeOf) { + return maybe ? maybe->sizeOfExcludingThis(mallocSizeOf) : 0; +} + +} // namespace wasm +} // namespace js + +#endif // wasm_utility_h diff --git a/js/src/wasm/WasmValType.cpp b/js/src/wasm/WasmValType.cpp new file mode 100644 index 0000000000..3265dae27f --- /dev/null +++ b/js/src/wasm/WasmValType.cpp @@ -0,0 +1,398 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmValType.h" + +#include "js/Conversions.h" +#include "js/ErrorReport.h" +#include "js/friend/ErrorMessages.h" // JSMSG_* +#include "js/Printf.h" +#include "js/Value.h" + +#include "vm/JSAtom.h" +#include "vm/JSObject.h" +#include "vm/StringType.h" +#include "wasm/WasmJS.h" + +#include "vm/JSAtom-inl.h" +#include "vm/JSObject-inl.h" + +using namespace js; +using namespace js::wasm; + +RefType RefType::topType() const { + switch (kind()) { + case RefType::Any: + case RefType::Eq: + case RefType::Array: + case RefType::Struct: + case RefType::None: + return RefType::any(); + case RefType::Func: + case RefType::NoFunc: + return RefType::func(); + case RefType::Extern: + case RefType::NoExtern: + return RefType::extern_(); + case RefType::TypeRef: + switch (typeDef()->kind()) { + case TypeDefKind::Array: + case TypeDefKind::Struct: + return RefType::any(); + case TypeDefKind::Func: + return RefType::func(); + case TypeDefKind::None: + MOZ_CRASH("should not see TypeDefKind::None at this point"); + } + } + MOZ_CRASH("switch is exhaustive"); +} + +TypeDefKind RefType::typeDefKind() const { + switch (kind()) { + case RefType::Struct: + return TypeDefKind::Struct; + case RefType::Array: + return TypeDefKind::Array; + case RefType::Func: + return TypeDefKind::Func; + default: + return TypeDefKind::None; + } + MOZ_CRASH("switch is exhaustive"); +} + +static bool ToRefType(JSContext* cx, JSLinearString* typeLinearStr, + RefType* out) { + if (StringEqualsLiteral(typeLinearStr, "anyfunc") || + StringEqualsLiteral(typeLinearStr, "funcref")) { + // The JS API uses "anyfunc" uniformly as the external name of funcref. We + // also allow "funcref" for compatibility with code we've already shipped. + *out = RefType::func(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "externref")) { + *out = RefType::extern_(); + return true; + } +#ifdef ENABLE_WASM_GC + if (GcAvailable(cx)) { + if (StringEqualsLiteral(typeLinearStr, "anyref")) { + *out = RefType::any(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "eqref")) { + *out = RefType::eq(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "structref")) { + *out = RefType::struct_(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "arrayref")) { + *out = RefType::array(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "nullfuncref")) { + *out = RefType::nofunc(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "nullexternref")) { + *out = RefType::noextern(); + return true; + } + if (StringEqualsLiteral(typeLinearStr, "nullref")) { + *out = RefType::none(); + return true; + } + } +#endif + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_STRING_VAL_TYPE); + return false; +} + +enum class RefTypeResult { + Failure, + Parsed, + Unparsed, +}; + +static RefTypeResult MaybeToRefType(JSContext* cx, HandleObject obj, + RefType* out) { +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + if (!wasm::FunctionReferencesAvailable(cx)) { + return RefTypeResult::Unparsed; + } + + JSAtom* refAtom = Atomize(cx, "ref", strlen("ref")); + if (!refAtom) { + return RefTypeResult::Failure; + } + RootedId refId(cx, AtomToId(refAtom)); + + RootedValue refVal(cx); + if (!GetProperty(cx, obj, obj, refId, &refVal)) { + return RefTypeResult::Failure; + } + + RootedString typeStr(cx, ToString(cx, refVal)); + if (!typeStr) { + return RefTypeResult::Failure; + } + + Rooted typeLinearStr(cx, typeStr->ensureLinear(cx)); + if (!typeLinearStr) { + return RefTypeResult::Failure; + } + + if (StringEqualsLiteral(typeLinearStr, "func")) { + *out = RefType::func(); + } else if (StringEqualsLiteral(typeLinearStr, "extern")) { + *out = RefType::extern_(); +# ifdef ENABLE_WASM_GC + } else if (GcAvailable(cx) && StringEqualsLiteral(typeLinearStr, "any")) { + *out = RefType::any(); + } else if (GcAvailable(cx) && StringEqualsLiteral(typeLinearStr, "eq")) { + *out = RefType::eq(); + } else if (GcAvailable(cx) && StringEqualsLiteral(typeLinearStr, "struct")) { + *out = RefType::struct_(); + } else if (GcAvailable(cx) && StringEqualsLiteral(typeLinearStr, "array")) { + *out = RefType::array(); +# endif + } else { + return RefTypeResult::Unparsed; + } + + JSAtom* nullableAtom = Atomize(cx, "nullable", strlen("nullable")); + if (!nullableAtom) { + return RefTypeResult::Failure; + } + RootedId nullableId(cx, AtomToId(nullableAtom)); + RootedValue nullableVal(cx); + if (!GetProperty(cx, obj, obj, nullableId, &nullableVal)) { + return RefTypeResult::Failure; + } + + bool nullable = ToBoolean(nullableVal); + if (!nullable) { + *out = out->asNonNullable(); + } + MOZ_ASSERT(out->isNullable() == nullable); + return RefTypeResult::Parsed; +#else + return RefTypeResult::Unparsed; +#endif +} + +bool wasm::ToValType(JSContext* cx, HandleValue v, ValType* out) { + if (v.isObject()) { + RootedObject obj(cx, &v.toObject()); + RefType refType; + switch (MaybeToRefType(cx, obj, &refType)) { + case RefTypeResult::Failure: + return false; + case RefTypeResult::Parsed: + *out = ValType(refType); + return true; + case RefTypeResult::Unparsed: + break; + } + } + + RootedString typeStr(cx, ToString(cx, v)); + if (!typeStr) { + return false; + } + + Rooted typeLinearStr(cx, typeStr->ensureLinear(cx)); + if (!typeLinearStr) { + return false; + } + + if (StringEqualsLiteral(typeLinearStr, "i32")) { + *out = ValType::I32; + } else if (StringEqualsLiteral(typeLinearStr, "i64")) { + *out = ValType::I64; + } else if (StringEqualsLiteral(typeLinearStr, "f32")) { + *out = ValType::F32; + } else if (StringEqualsLiteral(typeLinearStr, "f64")) { + *out = ValType::F64; +#ifdef ENABLE_WASM_SIMD + } else if (SimdAvailable(cx) && StringEqualsLiteral(typeLinearStr, "v128")) { + *out = ValType::V128; +#endif + } else { + RefType rt; + if (ToRefType(cx, typeLinearStr, &rt)) { + *out = ValType(rt); + } else { + // ToRefType will report an error when it fails, just return false + return false; + } + } + + return true; +} + +bool wasm::ToRefType(JSContext* cx, HandleValue v, RefType* out) { + if (v.isObject()) { + RootedObject obj(cx, &v.toObject()); + switch (MaybeToRefType(cx, obj, out)) { + case RefTypeResult::Failure: + return false; + case RefTypeResult::Parsed: + return true; + case RefTypeResult::Unparsed: + break; + } + } + + RootedString typeStr(cx, ToString(cx, v)); + if (!typeStr) { + return false; + } + + Rooted typeLinearStr(cx, typeStr->ensureLinear(cx)); + if (!typeLinearStr) { + return false; + } + + return ToRefType(cx, typeLinearStr, out); +} + +UniqueChars wasm::ToString(RefType type, const TypeContext* types) { + // Try to emit a shorthand version first + if (type.isNullable() && !type.isTypeRef()) { + const char* literal = nullptr; + switch (type.kind()) { + case RefType::Func: + literal = "funcref"; + break; + case RefType::Extern: + literal = "externref"; + break; + case RefType::Any: + literal = "anyref"; + break; + case RefType::NoFunc: + literal = "nullfuncref"; + break; + case RefType::NoExtern: + literal = "nullexternref"; + break; + case RefType::None: + literal = "nullref"; + break; + case RefType::Eq: + literal = "eqref"; + break; + case RefType::Struct: + literal = "structref"; + break; + case RefType::Array: + literal = "arrayref"; + break; + case RefType::TypeRef: { + MOZ_CRASH("type ref should not be possible here"); + } + } + return DuplicateString(literal); + } + + // Emit the full reference type with heap type + const char* heapType = nullptr; + switch (type.kind()) { + case RefType::Func: + heapType = "func"; + break; + case RefType::Extern: + heapType = "extern"; + break; + case RefType::Any: + heapType = "any"; + break; + case RefType::NoFunc: + heapType = "nofunc"; + break; + case RefType::NoExtern: + heapType = "noextern"; + break; + case RefType::None: + heapType = "none"; + break; + case RefType::Eq: + heapType = "eq"; + break; + case RefType::Struct: + heapType = "struct"; + break; + case RefType::Array: + heapType = "array"; + break; + case RefType::TypeRef: { + if (types) { + uint32_t typeIndex = types->indexOf(*type.typeDef()); + return JS_smprintf("(ref %s%d)", type.isNullable() ? "null " : "", + typeIndex); + } + return JS_smprintf("(ref %s?)", type.isNullable() ? "null " : ""); + } + } + return JS_smprintf("(ref %s%s)", type.isNullable() ? "null " : "", heapType); +} + +UniqueChars wasm::ToString(ValType type, const TypeContext* types) { + return ToString(type.fieldType(), types); +} + +UniqueChars wasm::ToString(FieldType type, const TypeContext* types) { + const char* literal = nullptr; + switch (type.kind()) { + case FieldType::I8: + literal = "i8"; + break; + case FieldType::I16: + literal = "i16"; + break; + case FieldType::I32: + literal = "i32"; + break; + case FieldType::I64: + literal = "i64"; + break; + case FieldType::V128: + literal = "v128"; + break; + case FieldType::F32: + literal = "f32"; + break; + case FieldType::F64: + literal = "f64"; + break; + case FieldType::Ref: + return ToString(type.refType(), types); + } + return DuplicateString(literal); +} + +UniqueChars wasm::ToString(const Maybe& type, + const TypeContext* types) { + return type ? ToString(type.ref(), types) : JS_smprintf("%s", "void"); +} diff --git a/js/src/wasm/WasmValType.h b/js/src/wasm/WasmValType.h new file mode 100644 index 0000000000..e0fdd43249 --- /dev/null +++ b/js/src/wasm/WasmValType.h @@ -0,0 +1,890 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_valtype_h +#define wasm_valtype_h + +#include "mozilla/HashTable.h" +#include "mozilla/Maybe.h" + +#include + +#include "jit/IonTypes.h" +#include "wasm/WasmConstants.h" +#include "wasm/WasmSerialize.h" +#include "wasm/WasmTypeDecls.h" + +namespace js { +namespace wasm { + +using mozilla::Maybe; + +class RecGroup; +class TypeDef; +class TypeContext; +enum class TypeDefKind : uint8_t; + +// A PackedTypeCode represents any value type. +union PackedTypeCode { + public: + using PackedRepr = uint64_t; + + private: + static constexpr size_t NullableBits = 1; + static constexpr size_t TypeCodeBits = 8; + static constexpr size_t TypeDefBits = 48; + static constexpr size_t PointerTagBits = 2; + + static_assert(NullableBits + TypeCodeBits + TypeDefBits + PointerTagBits <= + (sizeof(PackedRepr) * 8), + "enough bits"); + + PackedRepr bits_; + struct { + PackedRepr nullable_ : NullableBits; + PackedRepr typeCode_ : TypeCodeBits; + // A pointer to the TypeDef this type references. We use 48-bits for this, + // and rely on system memory allocators not allocating outside of this + // range. This is also assumed by JS::Value, and so should be safe here. + PackedRepr typeDef_ : TypeDefBits; + // Reserve the bottom two bits for use as a tagging scheme for BlockType + // and ResultType, which can encode a ValType inside themselves in special + // cases. + PackedRepr pointerTag_ : PointerTagBits; + }; + + public: + static constexpr PackedRepr NoTypeCode = ((uint64_t)1 << TypeCodeBits) - 1; + + static PackedTypeCode invalid() { + PackedTypeCode ptc = {}; + ptc.typeCode_ = NoTypeCode; + return ptc; + } + + static constexpr PackedTypeCode fromBits(PackedRepr bits) { + PackedTypeCode ptc = {}; + ptc.bits_ = bits; + return ptc; + } + + static PackedTypeCode pack(TypeCode tc, const TypeDef* typeDef, + bool isNullable) { + MOZ_ASSERT(uint32_t(tc) <= ((1 << TypeCodeBits) - 1)); + MOZ_ASSERT_IF(tc != AbstractTypeRefCode, typeDef == nullptr); + MOZ_ASSERT_IF(tc == AbstractTypeRefCode, typeDef != nullptr); + // Double check that the type definition was allocated within 48-bits, as + // noted above. + MOZ_ASSERT((uint64_t)typeDef <= ((uint64_t)1 << TypeDefBits) - 1); + PackedTypeCode ptc = {}; + ptc.typeCode_ = PackedRepr(tc); + ptc.typeDef_ = (uintptr_t)typeDef; + ptc.nullable_ = isNullable; + return ptc; + } + + static PackedTypeCode pack(TypeCode tc, bool nullable) { + return pack(tc, nullptr, nullable); + } + + static PackedTypeCode pack(TypeCode tc) { return pack(tc, nullptr, false); } + + bool isValid() const { return typeCode_ != NoTypeCode; } + + PackedRepr bits() const { return bits_; } + + TypeCode typeCode() const { + MOZ_ASSERT(isValid()); + return TypeCode(typeCode_); + } + + // Return the TypeCode, but return AbstractReferenceTypeCode for any reference + // type. + // + // This function is very, very hot, hence what would normally be a switch on + // the value `c` to map the reference types to AbstractReferenceTypeCode has + // been distilled into a simple comparison; this is fastest. Should type + // codes become too complicated for this to work then a lookup table also has + // better performance than a switch. + // + // An alternative is for the PackedTypeCode to represent something closer to + // what ValType needs, so that this decoding step is not necessary, but that + // moves complexity elsewhere, and the perf gain here would be only about 1% + // for baseline compilation throughput. + TypeCode typeCodeAbstracted() const { + TypeCode tc = typeCode(); + return tc < LowestPrimitiveTypeCode ? AbstractReferenceTypeCode : tc; + } + + // Return whether this type is a reference type. + bool isRefType() const { + return typeCodeAbstracted() == AbstractReferenceTypeCode; + } + + // Return whether this type is represented by a reference at runtime. + bool isRefRepr() const { return typeCode() < LowestPrimitiveTypeCode; } + + const TypeDef* typeDef() const { + MOZ_ASSERT(isValid()); + return (const TypeDef*)(uintptr_t)typeDef_; + } + + bool isNullable() const { + MOZ_ASSERT(isValid()); + return bool(nullable_); + } + + PackedTypeCode withIsNullable(bool nullable) const { + MOZ_ASSERT(isRefType()); + PackedTypeCode mutated = *this; + mutated.nullable_ = (PackedRepr)nullable; + return mutated; + } + + bool operator==(const PackedTypeCode& rhs) const { + return bits_ == rhs.bits_; + } + bool operator!=(const PackedTypeCode& rhs) const { + return bits_ != rhs.bits_; + } +}; + +static_assert(sizeof(PackedTypeCode) == sizeof(uint64_t), "packed"); + +// A SerializableTypeCode represents any value type in a form that can be +// serialized and deserialized. +union SerializableTypeCode { + using PackedRepr = uintptr_t; + + static constexpr size_t NullableBits = 1; + static constexpr size_t TypeCodeBits = 8; + static constexpr size_t TypeIndexBits = 20; + + PackedRepr bits; + struct { + PackedRepr nullable : NullableBits; + PackedRepr typeCode : TypeCodeBits; + PackedRepr typeIndex : TypeIndexBits; + }; + + WASM_CHECK_CACHEABLE_POD(bits); + + static constexpr PackedRepr NoTypeIndex = (1 << TypeIndexBits) - 1; + + static_assert(NullableBits + TypeCodeBits + TypeIndexBits <= + (sizeof(PackedRepr) * 8), + "enough bits"); + static_assert(NoTypeIndex < (1 << TypeIndexBits), "enough bits"); + static_assert(MaxTypes < NoTypeIndex, "enough bits"); + + // Defined in WasmSerialize.cpp + static inline SerializableTypeCode serialize(PackedTypeCode ptc, + const TypeContext& types); + inline PackedTypeCode deserialize(const TypeContext& types); +}; + +WASM_DECLARE_CACHEABLE_POD(SerializableTypeCode); +static_assert(sizeof(SerializableTypeCode) == sizeof(uintptr_t), "packed"); + +// [SMDOC] Matching type definitions +// +// WebAssembly type equality is structural, and we implement canonicalization +// such that equality of pointers to type definitions means that the type +// definitions are structurally equal. +// +// 'Matching' is the algorithm used to determine if two types are equal while +// canonicalizing types. +// +// A match type code encodes a type code for use in equality and hashing +// matching. It normalizes type references that are local to a recursion group +// so that they can be bitwise compared to type references from other recursion +// groups. +// +// This is useful for the following example: +// (rec (func $a)) +// (rec +// (func $b) +// (struct +// (field (ref $a))) +// (field (ref $b))) +// ) +// (rec +// (func $c) +// (struct +// (field (ref $a))) +// (field (ref $c))) +// ) +// +// The last two recursion groups are identical and should canonicalize to the +// same instance. However, they will be initially represented as two separate +// recursion group instances each with an array type instance with element +// types that point to the function type instance before them. A bitwise +// comparison of the element type pointers would fail. +// +// To solve this, we use `MatchTypeCode` to convert the example to: +// (rec (func $a)) +// (rec +// (func $b) +// (struct +// (field (ref nonlocal $a))) +// (field (ref local 0))) +// ) +// (rec +// (func $c) +// (struct +// (field (ref nonlocal $a))) +// (field (ref local 0))) +// ) +// +// Now, comparing the element types will see that these are local type +// references of the same kinds. `MatchTypeCode` performs the same mechanism +// as `tie` in the MVP presentation of type equality [1]. +// +// [1] +// https://github.com/WebAssembly/gc/blob/main/proposals/gc/MVP.md#equivalence +union MatchTypeCode { + using PackedRepr = uint64_t; + + static constexpr size_t NullableBits = 1; + static constexpr size_t TypeCodeBits = 8; + static constexpr size_t TypeRefBits = 48; + + PackedRepr bits; + struct { + PackedRepr nullable : NullableBits; + PackedRepr typeCode : TypeCodeBits; + PackedRepr typeRef : TypeRefBits; + }; + + WASM_CHECK_CACHEABLE_POD(bits); + + static_assert(NullableBits + TypeCodeBits + TypeRefBits <= + (sizeof(PackedRepr) * 8), + "enough bits"); + + // Defined in WasmTypeDef.h to avoid a cycle while allowing inlining + static inline MatchTypeCode forMatch(PackedTypeCode ptc, + const RecGroup* recGroup); + + bool operator==(MatchTypeCode other) const { return bits == other.bits; } + bool operator!=(MatchTypeCode other) const { return bits != other.bits; } + HashNumber hash() const { return HashNumber(bits); } +}; + +// An enum that describes the representation classes for tables; The table +// element type is mapped into this by Table::repr(). + +enum class TableRepr { Ref, Func }; + +// An enum that describes the different type hierarchies. + +enum class RefTypeHierarchy { Func, Extern, Any }; + +// The RefType carries more information about types t for which t.isRefType() +// is true. + +class RefType { + public: + enum Kind { + Func = uint8_t(TypeCode::FuncRef), + Extern = uint8_t(TypeCode::ExternRef), + Any = uint8_t(TypeCode::AnyRef), + NoFunc = uint8_t(TypeCode::NullFuncRef), + NoExtern = uint8_t(TypeCode::NullExternRef), + None = uint8_t(TypeCode::NullAnyRef), + Eq = uint8_t(TypeCode::EqRef), + Struct = uint8_t(TypeCode::StructRef), + Array = uint8_t(TypeCode::ArrayRef), + TypeRef = uint8_t(AbstractTypeRefCode) + }; + + private: + PackedTypeCode ptc_; + + RefType(Kind kind, bool nullable) + : ptc_(PackedTypeCode::pack(TypeCode(kind), nullable)) { + MOZ_ASSERT(isValid()); + } + + RefType(const TypeDef* typeDef, bool nullable) + : ptc_(PackedTypeCode::pack(AbstractTypeRefCode, typeDef, nullable)) { + MOZ_ASSERT(isValid()); + } + + public: + RefType() : ptc_(PackedTypeCode::invalid()) {} + explicit RefType(PackedTypeCode ptc) : ptc_(ptc) { MOZ_ASSERT(isValid()); } + + static RefType fromTypeCode(TypeCode tc, bool nullable) { + MOZ_ASSERT(tc != AbstractTypeRefCode); + return RefType(Kind(tc), nullable); + } + + static RefType fromTypeDef(const TypeDef* typeDef, bool nullable) { + return RefType(typeDef, nullable); + } + + Kind kind() const { return Kind(ptc_.typeCode()); } + + const TypeDef* typeDef() const { return ptc_.typeDef(); } + + PackedTypeCode packed() const { return ptc_; } + PackedTypeCode* addressOfPacked() { return &ptc_; } + const PackedTypeCode* addressOfPacked() const { return &ptc_; } + +#ifdef DEBUG + bool isValid() const { + MOZ_ASSERT((ptc_.typeCode() == AbstractTypeRefCode) == + (ptc_.typeDef() != nullptr)); + switch (ptc_.typeCode()) { + case TypeCode::FuncRef: + case TypeCode::ExternRef: + case TypeCode::AnyRef: + case TypeCode::EqRef: + case TypeCode::StructRef: + case TypeCode::ArrayRef: + case TypeCode::NullFuncRef: + case TypeCode::NullExternRef: + case TypeCode::NullAnyRef: + case AbstractTypeRefCode: + return true; + default: + return false; + } + } +#endif + + static RefType func() { return RefType(Func, true); } + static RefType extern_() { return RefType(Extern, true); } + static RefType any() { return RefType(Any, true); } + static RefType nofunc() { return RefType(NoFunc, true); } + static RefType noextern() { return RefType(NoExtern, true); } + static RefType none() { return RefType(None, true); } + static RefType eq() { return RefType(Eq, true); } + static RefType struct_() { return RefType(Struct, true); } + static RefType array() { return RefType(Array, true); } + + bool isFunc() const { return kind() == RefType::Func; } + bool isExtern() const { return kind() == RefType::Extern; } + bool isAny() const { return kind() == RefType::Any; } + bool isNoFunc() const { return kind() == RefType::NoFunc; } + bool isNoExtern() const { return kind() == RefType::NoExtern; } + bool isNone() const { return kind() == RefType::None; } + bool isEq() const { return kind() == RefType::Eq; } + bool isStruct() const { return kind() == RefType::Struct; } + bool isArray() const { return kind() == RefType::Array; } + bool isTypeRef() const { return kind() == RefType::TypeRef; } + + bool isNullable() const { return bool(ptc_.isNullable()); } + RefType asNonNullable() const { return withIsNullable(false); } + RefType withIsNullable(bool nullable) const { + return RefType(ptc_.withIsNullable(nullable)); + } + + bool isRefBottom() const { return isNone() || isNoFunc() || isNoExtern(); } + + // These methods are defined in WasmTypeDef.h to avoid a cycle while allowing + // inlining. + inline RefTypeHierarchy hierarchy() const; + inline TableRepr tableRepr() const; + inline bool isFuncHierarchy() const; + inline bool isExternHierarchy() const; + inline bool isAnyHierarchy() const; + static bool isSubTypeOf(RefType subType, RefType superType); + static bool castPossible(RefType sourceType, RefType destType); + + // Gets the top of the given type's hierarchy, e.g. Any for structs and + // arrays, and Func for funcs + RefType topType() const; + + // Gets the TypeDefKind associated with this RefType, e.g. TypeDefKind::Struct + // for RefType::Struct. + TypeDefKind typeDefKind() const; + + bool operator==(const RefType& that) const { return ptc_ == that.ptc_; } + bool operator!=(const RefType& that) const { return ptc_ != that.ptc_; } +}; + +class FieldTypeTraits { + public: + enum Kind { + I8 = uint8_t(TypeCode::I8), + I16 = uint8_t(TypeCode::I16), + I32 = uint8_t(TypeCode::I32), + I64 = uint8_t(TypeCode::I64), + F32 = uint8_t(TypeCode::F32), + F64 = uint8_t(TypeCode::F64), + V128 = uint8_t(TypeCode::V128), + Ref = uint8_t(AbstractReferenceTypeCode), + }; + + static bool isValidTypeCode(TypeCode tc) { + switch (tc) { +#ifdef ENABLE_WASM_GC + case TypeCode::I8: + case TypeCode::I16: +#endif + case TypeCode::I32: + case TypeCode::I64: + case TypeCode::F32: + case TypeCode::F64: +#ifdef ENABLE_WASM_SIMD + case TypeCode::V128: +#endif + case TypeCode::FuncRef: + case TypeCode::ExternRef: +#ifdef ENABLE_WASM_GC + case TypeCode::AnyRef: + case TypeCode::EqRef: + case TypeCode::StructRef: + case TypeCode::ArrayRef: + case TypeCode::NullFuncRef: + case TypeCode::NullExternRef: + case TypeCode::NullAnyRef: +#endif +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case AbstractTypeRefCode: +#endif + return true; + default: + return false; + } + } + + static bool isNumberTypeCode(TypeCode tc) { + switch (tc) { + case TypeCode::I32: + case TypeCode::I64: + case TypeCode::F32: + case TypeCode::F64: + return true; + default: + return false; + } + } + + static bool isPackedTypeCode(TypeCode tc) { + switch (tc) { +#ifdef ENABLE_WASM_GC + case TypeCode::I8: + case TypeCode::I16: + return true; +#endif + default: + return false; + } + } + + static bool isVectorTypeCode(TypeCode tc) { + switch (tc) { +#ifdef ENABLE_WASM_SIMD + case TypeCode::V128: + return true; +#endif + default: + return false; + } + } +}; + +class ValTypeTraits { + public: + enum Kind { + I32 = uint8_t(TypeCode::I32), + I64 = uint8_t(TypeCode::I64), + F32 = uint8_t(TypeCode::F32), + F64 = uint8_t(TypeCode::F64), + V128 = uint8_t(TypeCode::V128), + Ref = uint8_t(AbstractReferenceTypeCode), + }; + + static bool isValidTypeCode(TypeCode tc) { + switch (tc) { + case TypeCode::I32: + case TypeCode::I64: + case TypeCode::F32: + case TypeCode::F64: +#ifdef ENABLE_WASM_SIMD + case TypeCode::V128: +#endif + case TypeCode::FuncRef: + case TypeCode::ExternRef: +#ifdef ENABLE_WASM_GC + case TypeCode::AnyRef: + case TypeCode::EqRef: + case TypeCode::StructRef: + case TypeCode::ArrayRef: + case TypeCode::NullFuncRef: + case TypeCode::NullExternRef: + case TypeCode::NullAnyRef: +#endif +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case AbstractTypeRefCode: +#endif + return true; + default: + return false; + } + } + + static bool isNumberTypeCode(TypeCode tc) { + switch (tc) { + case TypeCode::I32: + case TypeCode::I64: + case TypeCode::F32: + case TypeCode::F64: + return true; + default: + return false; + } + } + + static bool isPackedTypeCode(TypeCode tc) { return false; } + + static bool isVectorTypeCode(TypeCode tc) { + switch (tc) { +#ifdef ENABLE_WASM_SIMD + case TypeCode::V128: + return true; +#endif + default: + return false; + } + } +}; + +// The PackedType represents the storage type of a WebAssembly location, whether +// parameter, local, field, or global. See specializations below for ValType and +// FieldType. + +template +class PackedType : public T { + public: + using Kind = typename T::Kind; + + protected: + PackedTypeCode tc_; + + explicit PackedType(TypeCode c) : tc_(PackedTypeCode::pack(c)) { + MOZ_ASSERT(c != AbstractTypeRefCode); + MOZ_ASSERT(isValid()); + } + + TypeCode typeCode() const { + MOZ_ASSERT(isValid()); + return tc_.typeCode(); + } + + public: + PackedType() : tc_(PackedTypeCode::invalid()) {} + + MOZ_IMPLICIT PackedType(Kind c) : tc_(PackedTypeCode::pack(TypeCode(c))) { + MOZ_ASSERT(c != Kind::Ref); + MOZ_ASSERT(isValid()); + } + + MOZ_IMPLICIT PackedType(RefType rt) : tc_(rt.packed()) { + MOZ_ASSERT(isValid()); + } + + explicit PackedType(PackedTypeCode ptc) : tc_(ptc) { MOZ_ASSERT(isValid()); } + + static PackedType fromMIRType(jit::MIRType mty) { + switch (mty) { + case jit::MIRType::Int32: + return PackedType::I32; + break; + case jit::MIRType::Int64: + return PackedType::I64; + break; + case jit::MIRType::Float32: + return PackedType::F32; + break; + case jit::MIRType::Double: + return PackedType::F64; + break; + case jit::MIRType::Simd128: + return PackedType::V128; + break; + case jit::MIRType::RefOrNull: + return PackedType::Ref; + default: + MOZ_CRASH("fromMIRType: unexpected type"); + } + } + + static PackedType fromNonRefTypeCode(TypeCode tc) { +#ifdef DEBUG + switch (tc) { + case TypeCode::I8: + case TypeCode::I16: + case TypeCode::I32: + case TypeCode::I64: + case TypeCode::F32: + case TypeCode::F64: + case TypeCode::V128: + break; + default: + MOZ_CRASH("Bad type code"); + } +#endif + return PackedType(tc); + } + + static PackedType fromBitsUnsafe(PackedTypeCode::PackedRepr bits) { + return PackedType(PackedTypeCode::fromBits(bits)); + } + + bool isValid() const { + if (!tc_.isValid()) { + return false; + } + return T::isValidTypeCode(tc_.typeCode()); + } + + MatchTypeCode forMatch(const RecGroup* recGroup) const { + return MatchTypeCode::forMatch(tc_, recGroup); + } + + PackedTypeCode packed() const { + MOZ_ASSERT(isValid()); + return tc_; + } + PackedTypeCode* addressOfPacked() { return &tc_; } + const PackedTypeCode* addressOfPacked() const { return &tc_; } + + PackedTypeCode::PackedRepr bitsUnsafe() const { + MOZ_ASSERT(isValid()); + return tc_.bits(); + } + + bool isNumber() const { return T::isNumberTypeCode(tc_.typeCode()); } + + bool isPacked() const { return T::isPackedTypeCode(tc_.typeCode()); } + + bool isVector() const { return T::isVectorTypeCode(tc_.typeCode()); } + + bool isRefType() const { return tc_.isRefType(); } + + bool isFuncRef() const { return tc_.typeCode() == TypeCode::FuncRef; } + + bool isExternRef() const { return tc_.typeCode() == TypeCode::ExternRef; } + + bool isAnyRef() const { return tc_.typeCode() == TypeCode::AnyRef; } + + bool isNoFunc() const { return tc_.typeCode() == TypeCode::NullFuncRef; } + + bool isNoExtern() const { return tc_.typeCode() == TypeCode::NullExternRef; } + + bool isNone() const { return tc_.typeCode() == TypeCode::NullAnyRef; } + + bool isEqRef() const { return tc_.typeCode() == TypeCode::EqRef; } + + bool isStructRef() const { return tc_.typeCode() == TypeCode::StructRef; } + + bool isArrayRef() const { return tc_.typeCode() == TypeCode::ArrayRef; } + + bool isTypeRef() const { return tc_.typeCode() == AbstractTypeRefCode; } + + bool isRefRepr() const { return tc_.isRefRepr(); } + + // Returns whether the type has a default value. + bool isDefaultable() const { return !(isRefType() && !isNullable()); } + + // Returns whether the type has a representation in JS. + bool isExposable() const { +#if defined(ENABLE_WASM_SIMD) + return kind() != Kind::V128; +#else + return true; +#endif + } + + bool isNullable() const { return tc_.isNullable(); } + + const TypeDef* typeDef() const { return tc_.typeDef(); } + + Kind kind() const { return Kind(tc_.typeCodeAbstracted()); } + + RefType refType() const { + MOZ_ASSERT(isRefType()); + return RefType(tc_); + } + + RefType::Kind refTypeKind() const { + MOZ_ASSERT(isRefType()); + return RefType(tc_).kind(); + } + + // Some types are encoded as JS::Value when they escape from Wasm (when passed + // as parameters to imports or returned from exports). For ExternRef the + // Value encoding is pretty much a requirement. For other types it's a choice + // that may (temporarily) simplify some code. + bool isEncodedAsJSValueOnEscape() const { return isRefType(); } + + uint32_t size() const { + switch (tc_.typeCodeAbstracted()) { + case TypeCode::I8: + return 1; + case TypeCode::I16: + return 2; + case TypeCode::I32: + return 4; + case TypeCode::I64: + return 8; + case TypeCode::F32: + return 4; + case TypeCode::F64: + return 8; + case TypeCode::V128: + return 16; + case AbstractReferenceTypeCode: + return sizeof(void*); + default: + MOZ_ASSERT_UNREACHABLE(); + return 0; + } + } + uint32_t alignmentInStruct() const { return size(); } + uint32_t indexingShift() const { + switch (size()) { + case 1: + return 0; + case 2: + return 1; + case 4: + return 2; + case 8: + return 3; + case 16: + return 4; + default: + MOZ_ASSERT_UNREACHABLE(); + return 0; + } + } + + PackedType widenToValType() const { + switch (tc_.typeCodeAbstracted()) { + case TypeCode::I8: + case TypeCode::I16: + return PackedType::I32; + default: + return PackedType(tc_); + } + } + + PackedType valType() const { + MOZ_ASSERT(isValType()); + return PackedType(tc_); + } + + // Note, ToMIRType is only correct within Wasm, where an AnyRef is represented + // as a pointer. At the JS/wasm boundary, an AnyRef can be represented as a + // JS::Value, and the type translation may have to be handled specially and on + // a case-by-case basis. + jit::MIRType toMIRType() const { + switch (tc_.typeCodeAbstracted()) { + case TypeCode::I32: + return jit::MIRType::Int32; + case TypeCode::I64: + return jit::MIRType::Int64; + case TypeCode::F32: + return jit::MIRType::Float32; + case TypeCode::F64: + return jit::MIRType::Double; + case TypeCode::V128: + return jit::MIRType::Simd128; + case AbstractReferenceTypeCode: + return jit::MIRType::RefOrNull; + default: + MOZ_CRASH("bad type"); + } + } + + bool isValType() const { + switch (tc_.typeCode()) { + case TypeCode::I8: + case TypeCode::I16: + return false; + default: + return true; + } + } + + PackedType fieldType() const { + MOZ_ASSERT(isValid()); + return PackedType(tc_); + } + + static bool isSubTypeOf(PackedType subType, PackedType superType) { + // Anything is a subtype of itself. + if (subType == superType) { + return true; + } + + // A reference may be a subtype of another reference + if (subType.isRefType() && superType.isRefType()) { + return RefType::isSubTypeOf(subType.refType(), superType.refType()); + } + + return false; + } + + bool operator==(const PackedType& that) const { + MOZ_ASSERT(isValid() && that.isValid()); + return tc_ == that.tc_; + } + + bool operator!=(const PackedType& that) const { + MOZ_ASSERT(isValid() && that.isValid()); + return tc_ != that.tc_; + } + + bool operator==(Kind that) const { + MOZ_ASSERT(isValid()); + MOZ_ASSERT(that != Kind::Ref); + return Kind(typeCode()) == that; + } + + bool operator!=(Kind that) const { return !(*this == that); } +}; + +using ValType = PackedType; +using FieldType = PackedType; + +// The dominant use of this data type is for locals and args, and profiling +// with ZenGarden and Tanks suggests an initial size of 16 minimises heap +// allocation, both in terms of blocks and bytes. +using ValTypeVector = Vector; + +// ValType utilities + +extern bool ToValType(JSContext* cx, HandleValue v, ValType* out); +extern bool ToRefType(JSContext* cx, HandleValue v, RefType* out); + +extern UniqueChars ToString(RefType type, const TypeContext* types); +extern UniqueChars ToString(ValType type, const TypeContext* types); +extern UniqueChars ToString(FieldType type, const TypeContext* types); +extern UniqueChars ToString(const Maybe& type, + const TypeContext* types); + +} // namespace wasm +} // namespace js + +#endif // wasm_valtype_h diff --git a/js/src/wasm/WasmValidate.cpp b/js/src/wasm/WasmValidate.cpp new file mode 100644 index 0000000000..eb7d7f04e0 --- /dev/null +++ b/js/src/wasm/WasmValidate.cpp @@ -0,0 +1,3223 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmValidate.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/Span.h" +#include "mozilla/Utf8.h" + +#include "js/Printf.h" +#include "js/String.h" // JS::MaxStringLength +#include "vm/JSContext.h" +#include "vm/Realm.h" +#include "wasm/WasmOpIter.h" + +using namespace js; +using namespace js::jit; +using namespace js::wasm; + +using mozilla::AsChars; +using mozilla::CheckedInt; +using mozilla::CheckedInt32; +using mozilla::IsUtf8; +using mozilla::Span; + +// Misc helpers. + +bool wasm::EncodeLocalEntries(Encoder& e, const ValTypeVector& locals) { + if (locals.length() > MaxLocals) { + return false; + } + + uint32_t numLocalEntries = 0; + if (locals.length()) { + ValType prev = locals[0]; + numLocalEntries++; + for (ValType t : locals) { + if (t != prev) { + numLocalEntries++; + prev = t; + } + } + } + + if (!e.writeVarU32(numLocalEntries)) { + return false; + } + + if (numLocalEntries) { + ValType prev = locals[0]; + uint32_t count = 1; + for (uint32_t i = 1; i < locals.length(); i++, count++) { + if (prev != locals[i]) { + if (!e.writeVarU32(count)) { + return false; + } + if (!e.writeValType(prev)) { + return false; + } + prev = locals[i]; + count = 0; + } + } + if (!e.writeVarU32(count)) { + return false; + } + if (!e.writeValType(prev)) { + return false; + } + } + + return true; +} + +bool wasm::DecodeLocalEntries(Decoder& d, const TypeContext& types, + const FeatureArgs& features, + ValTypeVector* locals) { + uint32_t numLocalEntries; + if (!d.readVarU32(&numLocalEntries)) { + return d.fail("failed to read number of local entries"); + } + + for (uint32_t i = 0; i < numLocalEntries; i++) { + uint32_t count; + if (!d.readVarU32(&count)) { + return d.fail("failed to read local entry count"); + } + + if (MaxLocals - locals->length() < count) { + return d.fail("too many locals"); + } + + ValType type; + if (!d.readValType(types, features, &type)) { + return false; + } + + if (!locals->appendN(type, count)) { + return false; + } + } + + return true; +} + +bool wasm::DecodeValidatedLocalEntries(const TypeContext& types, Decoder& d, + ValTypeVector* locals) { + uint32_t numLocalEntries; + MOZ_ALWAYS_TRUE(d.readVarU32(&numLocalEntries)); + + for (uint32_t i = 0; i < numLocalEntries; i++) { + uint32_t count = d.uncheckedReadVarU32(); + MOZ_ASSERT(MaxLocals - locals->length() >= count); + if (!locals->appendN(d.uncheckedReadValType(types), count)) { + return false; + } + } + + return true; +} + +bool wasm::CheckIsSubtypeOf(Decoder& d, const ModuleEnvironment& env, + size_t opcodeOffset, FieldType subType, + FieldType superType) { + if (FieldType::isSubTypeOf(subType, superType)) { + return true; + } + + UniqueChars subText = ToString(subType, env.types); + if (!subText) { + return false; + } + + UniqueChars superText = ToString(superType, env.types); + if (!superText) { + return false; + } + + UniqueChars error( + JS_smprintf("type mismatch: expression has type %s but expected %s", + subText.get(), superText.get())); + if (!error) { + return false; + } + + return d.fail(opcodeOffset, error.get()); +} + +// Function body validation. + +static bool DecodeFunctionBodyExprs(const ModuleEnvironment& env, + uint32_t funcIndex, + const ValTypeVector& locals, + const uint8_t* bodyEnd, Decoder* d) { + ValidatingOpIter iter(env, *d); + + if (!iter.startFunction(funcIndex, locals)) { + return false; + } + +#define CHECK(c) \ + if (!(c)) return false; \ + break + + while (true) { + OpBytes op; + if (!iter.readOp(&op)) { + return false; + } + + Nothing nothing; + NothingVector nothings{}; + ResultType unusedType; + + switch (op.b0) { + case uint16_t(Op::End): { + LabelKind unusedKind; + if (!iter.readEnd(&unusedKind, &unusedType, ¬hings, ¬hings)) { + return false; + } + iter.popEnd(); + if (iter.controlStackEmpty()) { + return iter.endFunction(bodyEnd); + } + break; + } + case uint16_t(Op::Nop): + CHECK(iter.readNop()); + case uint16_t(Op::Drop): + CHECK(iter.readDrop()); + case uint16_t(Op::Call): { + uint32_t unusedIndex; + NothingVector unusedArgs{}; + CHECK(iter.readCall(&unusedIndex, &unusedArgs)); + } + case uint16_t(Op::CallIndirect): { + uint32_t unusedIndex, unusedIndex2; + NothingVector unusedArgs{}; + CHECK(iter.readCallIndirect(&unusedIndex, &unusedIndex2, ¬hing, + &unusedArgs)); + } +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::CallRef): { + if (!env.functionReferencesEnabled()) { + return iter.unrecognizedOpcode(&op); + } + const FuncType* unusedType; + NothingVector unusedArgs{}; + CHECK(iter.readCallRef(&unusedType, ¬hing, &unusedArgs)); + } +#endif + case uint16_t(Op::I32Const): { + int32_t unused; + CHECK(iter.readI32Const(&unused)); + } + case uint16_t(Op::I64Const): { + int64_t unused; + CHECK(iter.readI64Const(&unused)); + } + case uint16_t(Op::F32Const): { + float unused; + CHECK(iter.readF32Const(&unused)); + } + case uint16_t(Op::F64Const): { + double unused; + CHECK(iter.readF64Const(&unused)); + } + case uint16_t(Op::LocalGet): { + uint32_t unused; + CHECK(iter.readGetLocal(locals, &unused)); + } + case uint16_t(Op::LocalSet): { + uint32_t unused; + CHECK(iter.readSetLocal(locals, &unused, ¬hing)); + } + case uint16_t(Op::LocalTee): { + uint32_t unused; + CHECK(iter.readTeeLocal(locals, &unused, ¬hing)); + } + case uint16_t(Op::GlobalGet): { + uint32_t unused; + CHECK(iter.readGetGlobal(&unused)); + } + case uint16_t(Op::GlobalSet): { + uint32_t unused; + CHECK(iter.readSetGlobal(&unused, ¬hing)); + } + case uint16_t(Op::TableGet): { + uint32_t unusedTableIndex; + CHECK(iter.readTableGet(&unusedTableIndex, ¬hing)); + } + case uint16_t(Op::TableSet): { + uint32_t unusedTableIndex; + CHECK(iter.readTableSet(&unusedTableIndex, ¬hing, ¬hing)); + } + case uint16_t(Op::SelectNumeric): { + StackType unused; + CHECK(iter.readSelect(/*typed*/ false, &unused, ¬hing, ¬hing, + ¬hing)); + } + case uint16_t(Op::SelectTyped): { + StackType unused; + CHECK(iter.readSelect(/*typed*/ true, &unused, ¬hing, ¬hing, + ¬hing)); + } + case uint16_t(Op::Block): + CHECK(iter.readBlock(&unusedType)); + case uint16_t(Op::Loop): + CHECK(iter.readLoop(&unusedType)); + case uint16_t(Op::If): + CHECK(iter.readIf(&unusedType, ¬hing)); + case uint16_t(Op::Else): + CHECK(iter.readElse(&unusedType, &unusedType, ¬hings)); + case uint16_t(Op::I32Clz): + case uint16_t(Op::I32Ctz): + case uint16_t(Op::I32Popcnt): + CHECK(iter.readUnary(ValType::I32, ¬hing)); + case uint16_t(Op::I64Clz): + case uint16_t(Op::I64Ctz): + case uint16_t(Op::I64Popcnt): + CHECK(iter.readUnary(ValType::I64, ¬hing)); + case uint16_t(Op::F32Abs): + case uint16_t(Op::F32Neg): + case uint16_t(Op::F32Ceil): + case uint16_t(Op::F32Floor): + case uint16_t(Op::F32Sqrt): + case uint16_t(Op::F32Trunc): + case uint16_t(Op::F32Nearest): + CHECK(iter.readUnary(ValType::F32, ¬hing)); + case uint16_t(Op::F64Abs): + case uint16_t(Op::F64Neg): + case uint16_t(Op::F64Ceil): + case uint16_t(Op::F64Floor): + case uint16_t(Op::F64Sqrt): + case uint16_t(Op::F64Trunc): + case uint16_t(Op::F64Nearest): + CHECK(iter.readUnary(ValType::F64, ¬hing)); + case uint16_t(Op::I32Add): + case uint16_t(Op::I32Sub): + case uint16_t(Op::I32Mul): + case uint16_t(Op::I32DivS): + case uint16_t(Op::I32DivU): + case uint16_t(Op::I32RemS): + case uint16_t(Op::I32RemU): + case uint16_t(Op::I32And): + case uint16_t(Op::I32Or): + case uint16_t(Op::I32Xor): + case uint16_t(Op::I32Shl): + case uint16_t(Op::I32ShrS): + case uint16_t(Op::I32ShrU): + case uint16_t(Op::I32Rotl): + case uint16_t(Op::I32Rotr): + CHECK(iter.readBinary(ValType::I32, ¬hing, ¬hing)); + case uint16_t(Op::I64Add): + case uint16_t(Op::I64Sub): + case uint16_t(Op::I64Mul): + case uint16_t(Op::I64DivS): + case uint16_t(Op::I64DivU): + case uint16_t(Op::I64RemS): + case uint16_t(Op::I64RemU): + case uint16_t(Op::I64And): + case uint16_t(Op::I64Or): + case uint16_t(Op::I64Xor): + case uint16_t(Op::I64Shl): + case uint16_t(Op::I64ShrS): + case uint16_t(Op::I64ShrU): + case uint16_t(Op::I64Rotl): + case uint16_t(Op::I64Rotr): + CHECK(iter.readBinary(ValType::I64, ¬hing, ¬hing)); + case uint16_t(Op::F32Add): + case uint16_t(Op::F32Sub): + case uint16_t(Op::F32Mul): + case uint16_t(Op::F32Div): + case uint16_t(Op::F32Min): + case uint16_t(Op::F32Max): + case uint16_t(Op::F32CopySign): + CHECK(iter.readBinary(ValType::F32, ¬hing, ¬hing)); + case uint16_t(Op::F64Add): + case uint16_t(Op::F64Sub): + case uint16_t(Op::F64Mul): + case uint16_t(Op::F64Div): + case uint16_t(Op::F64Min): + case uint16_t(Op::F64Max): + case uint16_t(Op::F64CopySign): + CHECK(iter.readBinary(ValType::F64, ¬hing, ¬hing)); + case uint16_t(Op::I32Eq): + case uint16_t(Op::I32Ne): + case uint16_t(Op::I32LtS): + case uint16_t(Op::I32LtU): + case uint16_t(Op::I32LeS): + case uint16_t(Op::I32LeU): + case uint16_t(Op::I32GtS): + case uint16_t(Op::I32GtU): + case uint16_t(Op::I32GeS): + case uint16_t(Op::I32GeU): + CHECK(iter.readComparison(ValType::I32, ¬hing, ¬hing)); + case uint16_t(Op::I64Eq): + case uint16_t(Op::I64Ne): + case uint16_t(Op::I64LtS): + case uint16_t(Op::I64LtU): + case uint16_t(Op::I64LeS): + case uint16_t(Op::I64LeU): + case uint16_t(Op::I64GtS): + case uint16_t(Op::I64GtU): + case uint16_t(Op::I64GeS): + case uint16_t(Op::I64GeU): + CHECK(iter.readComparison(ValType::I64, ¬hing, ¬hing)); + case uint16_t(Op::F32Eq): + case uint16_t(Op::F32Ne): + case uint16_t(Op::F32Lt): + case uint16_t(Op::F32Le): + case uint16_t(Op::F32Gt): + case uint16_t(Op::F32Ge): + CHECK(iter.readComparison(ValType::F32, ¬hing, ¬hing)); + case uint16_t(Op::F64Eq): + case uint16_t(Op::F64Ne): + case uint16_t(Op::F64Lt): + case uint16_t(Op::F64Le): + case uint16_t(Op::F64Gt): + case uint16_t(Op::F64Ge): + CHECK(iter.readComparison(ValType::F64, ¬hing, ¬hing)); + case uint16_t(Op::I32Eqz): + CHECK(iter.readConversion(ValType::I32, ValType::I32, ¬hing)); + case uint16_t(Op::I64Eqz): + case uint16_t(Op::I32WrapI64): + CHECK(iter.readConversion(ValType::I64, ValType::I32, ¬hing)); + case uint16_t(Op::I32TruncF32S): + case uint16_t(Op::I32TruncF32U): + case uint16_t(Op::I32ReinterpretF32): + CHECK(iter.readConversion(ValType::F32, ValType::I32, ¬hing)); + case uint16_t(Op::I32TruncF64S): + case uint16_t(Op::I32TruncF64U): + CHECK(iter.readConversion(ValType::F64, ValType::I32, ¬hing)); + case uint16_t(Op::I64ExtendI32S): + case uint16_t(Op::I64ExtendI32U): + CHECK(iter.readConversion(ValType::I32, ValType::I64, ¬hing)); + case uint16_t(Op::I64TruncF32S): + case uint16_t(Op::I64TruncF32U): + CHECK(iter.readConversion(ValType::F32, ValType::I64, ¬hing)); + case uint16_t(Op::I64TruncF64S): + case uint16_t(Op::I64TruncF64U): + case uint16_t(Op::I64ReinterpretF64): + CHECK(iter.readConversion(ValType::F64, ValType::I64, ¬hing)); + case uint16_t(Op::F32ConvertI32S): + case uint16_t(Op::F32ConvertI32U): + case uint16_t(Op::F32ReinterpretI32): + CHECK(iter.readConversion(ValType::I32, ValType::F32, ¬hing)); + case uint16_t(Op::F32ConvertI64S): + case uint16_t(Op::F32ConvertI64U): + CHECK(iter.readConversion(ValType::I64, ValType::F32, ¬hing)); + case uint16_t(Op::F32DemoteF64): + CHECK(iter.readConversion(ValType::F64, ValType::F32, ¬hing)); + case uint16_t(Op::F64ConvertI32S): + case uint16_t(Op::F64ConvertI32U): + CHECK(iter.readConversion(ValType::I32, ValType::F64, ¬hing)); + case uint16_t(Op::F64ConvertI64S): + case uint16_t(Op::F64ConvertI64U): + case uint16_t(Op::F64ReinterpretI64): + CHECK(iter.readConversion(ValType::I64, ValType::F64, ¬hing)); + case uint16_t(Op::F64PromoteF32): + CHECK(iter.readConversion(ValType::F32, ValType::F64, ¬hing)); + case uint16_t(Op::I32Extend8S): + case uint16_t(Op::I32Extend16S): + CHECK(iter.readConversion(ValType::I32, ValType::I32, ¬hing)); + case uint16_t(Op::I64Extend8S): + case uint16_t(Op::I64Extend16S): + case uint16_t(Op::I64Extend32S): + CHECK(iter.readConversion(ValType::I64, ValType::I64, ¬hing)); + case uint16_t(Op::I32Load8S): + case uint16_t(Op::I32Load8U): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I32, 1, &addr)); + } + case uint16_t(Op::I32Load16S): + case uint16_t(Op::I32Load16U): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I32, 2, &addr)); + } + case uint16_t(Op::I32Load): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I32, 4, &addr)); + } + case uint16_t(Op::I64Load8S): + case uint16_t(Op::I64Load8U): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I64, 1, &addr)); + } + case uint16_t(Op::I64Load16S): + case uint16_t(Op::I64Load16U): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I64, 2, &addr)); + } + case uint16_t(Op::I64Load32S): + case uint16_t(Op::I64Load32U): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I64, 4, &addr)); + } + case uint16_t(Op::I64Load): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::I64, 8, &addr)); + } + case uint16_t(Op::F32Load): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::F32, 4, &addr)); + } + case uint16_t(Op::F64Load): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::F64, 8, &addr)); + } + case uint16_t(Op::I32Store8): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I32, 1, &addr, ¬hing)); + } + case uint16_t(Op::I32Store16): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I32, 2, &addr, ¬hing)); + } + case uint16_t(Op::I32Store): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I32, 4, &addr, ¬hing)); + } + case uint16_t(Op::I64Store8): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I64, 1, &addr, ¬hing)); + } + case uint16_t(Op::I64Store16): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I64, 2, &addr, ¬hing)); + } + case uint16_t(Op::I64Store32): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I64, 4, &addr, ¬hing)); + } + case uint16_t(Op::I64Store): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::I64, 8, &addr, ¬hing)); + } + case uint16_t(Op::F32Store): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::F32, 4, &addr, ¬hing)); + } + case uint16_t(Op::F64Store): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::F64, 8, &addr, ¬hing)); + } + case uint16_t(Op::MemoryGrow): + CHECK(iter.readMemoryGrow(¬hing)); + case uint16_t(Op::MemorySize): + CHECK(iter.readMemorySize()); + case uint16_t(Op::Br): { + uint32_t unusedDepth; + CHECK(iter.readBr(&unusedDepth, &unusedType, ¬hings)); + } + case uint16_t(Op::BrIf): { + uint32_t unusedDepth; + CHECK(iter.readBrIf(&unusedDepth, &unusedType, ¬hings, ¬hing)); + } + case uint16_t(Op::BrTable): { + Uint32Vector unusedDepths; + uint32_t unusedDefault; + CHECK(iter.readBrTable(&unusedDepths, &unusedDefault, &unusedType, + ¬hings, ¬hing)); + } + case uint16_t(Op::Return): + CHECK(iter.readReturn(¬hings)); + case uint16_t(Op::Unreachable): + CHECK(iter.readUnreachable()); +#ifdef ENABLE_WASM_GC + case uint16_t(Op::GcPrefix): { + if (!env.gcEnabled()) { + return iter.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(GcOp::StructNew): { + uint32_t unusedUint; + NothingVector unusedArgs{}; + CHECK(iter.readStructNew(&unusedUint, &unusedArgs)); + } + case uint32_t(GcOp::StructNewDefault): { + uint32_t unusedUint; + CHECK(iter.readStructNewDefault(&unusedUint)); + } + case uint32_t(GcOp::StructGet): { + uint32_t unusedUint1, unusedUint2; + CHECK(iter.readStructGet(&unusedUint1, &unusedUint2, + FieldWideningOp::None, ¬hing)); + } + case uint32_t(GcOp::StructGetS): { + uint32_t unusedUint1, unusedUint2; + CHECK(iter.readStructGet(&unusedUint1, &unusedUint2, + FieldWideningOp::Signed, ¬hing)); + } + case uint32_t(GcOp::StructGetU): { + uint32_t unusedUint1, unusedUint2; + CHECK(iter.readStructGet(&unusedUint1, &unusedUint2, + FieldWideningOp::Unsigned, ¬hing)); + } + case uint32_t(GcOp::StructSet): { + uint32_t unusedUint1, unusedUint2; + CHECK(iter.readStructSet(&unusedUint1, &unusedUint2, ¬hing, + ¬hing)); + } + case uint32_t(GcOp::ArrayNew): { + uint32_t unusedUint; + CHECK(iter.readArrayNew(&unusedUint, ¬hing, ¬hing)); + } + case uint32_t(GcOp::ArrayNewFixed): { + uint32_t unusedUint1, unusedUint2; + CHECK( + iter.readArrayNewFixed(&unusedUint1, &unusedUint2, ¬hings)); + } + case uint32_t(GcOp::ArrayNewDefault): { + uint32_t unusedUint; + CHECK(iter.readArrayNewDefault(&unusedUint, ¬hing)); + } + case uint32_t(GcOp::ArrayNewData): { + uint32_t unusedUint1, unusedUint2; + CHECK(iter.readArrayNewData(&unusedUint1, &unusedUint2, ¬hing, + ¬hing)); + } + case uint32_t(GcOp::ArrayInitFromElemStaticV5): + case uint32_t(GcOp::ArrayNewElem): { + uint32_t unusedUint1, unusedUint2; + CHECK(iter.readArrayNewElem(&unusedUint1, &unusedUint2, ¬hing, + ¬hing)); + } + case uint32_t(GcOp::ArrayGet): { + uint32_t unusedUint1; + CHECK(iter.readArrayGet(&unusedUint1, FieldWideningOp::None, + ¬hing, ¬hing)); + } + case uint32_t(GcOp::ArrayGetS): { + uint32_t unusedUint1; + CHECK(iter.readArrayGet(&unusedUint1, FieldWideningOp::Signed, + ¬hing, ¬hing)); + } + case uint32_t(GcOp::ArrayGetU): { + uint32_t unusedUint1; + CHECK(iter.readArrayGet(&unusedUint1, FieldWideningOp::Unsigned, + ¬hing, ¬hing)); + } + case uint32_t(GcOp::ArraySet): { + uint32_t unusedUint1; + CHECK( + iter.readArraySet(&unusedUint1, ¬hing, ¬hing, ¬hing)); + } + case uint32_t(GcOp::ArrayLenWithTypeIndex): { + CHECK(iter.readArrayLen(/*decodeIgnoredTypeIndex=*/true, ¬hing)); + } + case uint32_t(GcOp::ArrayLen): { + CHECK( + iter.readArrayLen(/*decodeIgnoredTypeIndex=*/false, ¬hing)); + } + case uint32_t(GcOp::ArrayCopy): { + int32_t unusedInt; + bool unusedBool; + CHECK(iter.readArrayCopy(&unusedInt, &unusedBool, ¬hing, + ¬hing, ¬hing, ¬hing, ¬hing)); + } + case uint16_t(GcOp::RefTestV5): { + RefType unusedSourceType; + uint32_t unusedTypeIndex; + CHECK(iter.readRefTestV5(&unusedSourceType, &unusedTypeIndex, + ¬hing)); + } + case uint16_t(GcOp::RefCastV5): { + RefType unusedSourceType; + uint32_t unusedTypeIndex; + CHECK(iter.readRefCastV5(&unusedSourceType, &unusedTypeIndex, + ¬hing)); + } + case uint16_t(GcOp::RefTest): { + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readRefTest(false, &unusedSourceType, &unusedDestType, + ¬hing)); + } + case uint16_t(GcOp::RefTestNull): { + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readRefTest(true, &unusedSourceType, &unusedDestType, + ¬hing)); + } + case uint16_t(GcOp::RefCast): { + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readRefCast(false, &unusedSourceType, &unusedDestType, + ¬hing)); + } + case uint16_t(GcOp::RefCastNull): { + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readRefCast(true, &unusedSourceType, &unusedDestType, + ¬hing)); + } + case uint16_t(GcOp::BrOnCast): { + bool unusedOnSuccess; + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readBrOnCast(&unusedOnSuccess, &unusedRelativeDepth, + &unusedSourceType, &unusedDestType, + &unusedType, ¬hings)); + } + case uint16_t(GcOp::BrOnCastV5): { + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + uint32_t typeIndex; + CHECK(iter.readBrOnCastV5(&unusedRelativeDepth, &unusedSourceType, + &typeIndex, &unusedType, ¬hings)); + } + case uint16_t(GcOp::BrOnCastFailV5): { + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + uint32_t typeIndex; + CHECK(iter.readBrOnCastFailV5(&unusedRelativeDepth, + &unusedSourceType, &typeIndex, + &unusedType, ¬hings)); + } + case uint16_t(GcOp::BrOnCastHeapV5): { + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readBrOnCastHeapV5(false, &unusedRelativeDepth, + &unusedSourceType, &unusedDestType, + &unusedType, ¬hings)); + } + case uint16_t(GcOp::BrOnCastHeapNullV5): { + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readBrOnCastHeapV5(true, &unusedRelativeDepth, + &unusedSourceType, &unusedDestType, + &unusedType, ¬hings)); + } + case uint16_t(GcOp::BrOnCastFailHeapV5): { + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readBrOnCastFailHeapV5( + false, &unusedRelativeDepth, &unusedSourceType, &unusedDestType, + &unusedType, ¬hings)); + } + case uint16_t(GcOp::BrOnCastFailHeapNullV5): { + uint32_t unusedRelativeDepth; + RefType unusedSourceType; + RefType unusedDestType; + CHECK(iter.readBrOnCastFailHeapV5( + true, &unusedRelativeDepth, &unusedSourceType, &unusedDestType, + &unusedType, ¬hings)); + } + case uint16_t(GcOp::RefAsStructV5): { + CHECK(iter.readConversion( + ValType(RefType::any()), + ValType(RefType::struct_().asNonNullable()), ¬hing)); + } + case uint16_t(GcOp::BrOnNonStructV5): { + uint32_t unusedRelativeDepth; + CHECK(iter.readBrOnNonStructV5(&unusedRelativeDepth, &unusedType, + ¬hings)); + } + case uint16_t(GcOp::ExternInternalize): { + CHECK(iter.readRefConversion(RefType::extern_(), RefType::any(), + ¬hing)); + } + case uint16_t(GcOp::ExternExternalize): { + CHECK(iter.readRefConversion(RefType::any(), RefType::extern_(), + ¬hing)); + } + default: + return iter.unrecognizedOpcode(&op); + } + break; + } +#endif + +#ifdef ENABLE_WASM_SIMD + case uint16_t(Op::SimdPrefix): { + if (!env.simdAvailable()) { + return iter.unrecognizedOpcode(&op); + } + uint32_t noIndex; + switch (op.b1) { + case uint32_t(SimdOp::I8x16ExtractLaneS): + case uint32_t(SimdOp::I8x16ExtractLaneU): + CHECK(iter.readExtractLane(ValType::I32, 16, &noIndex, ¬hing)); + case uint32_t(SimdOp::I16x8ExtractLaneS): + case uint32_t(SimdOp::I16x8ExtractLaneU): + CHECK(iter.readExtractLane(ValType::I32, 8, &noIndex, ¬hing)); + case uint32_t(SimdOp::I32x4ExtractLane): + CHECK(iter.readExtractLane(ValType::I32, 4, &noIndex, ¬hing)); + case uint32_t(SimdOp::I64x2ExtractLane): + CHECK(iter.readExtractLane(ValType::I64, 2, &noIndex, ¬hing)); + case uint32_t(SimdOp::F32x4ExtractLane): + CHECK(iter.readExtractLane(ValType::F32, 4, &noIndex, ¬hing)); + case uint32_t(SimdOp::F64x2ExtractLane): + CHECK(iter.readExtractLane(ValType::F64, 2, &noIndex, ¬hing)); + + case uint32_t(SimdOp::I8x16Splat): + case uint32_t(SimdOp::I16x8Splat): + case uint32_t(SimdOp::I32x4Splat): + CHECK(iter.readConversion(ValType::I32, ValType::V128, ¬hing)); + case uint32_t(SimdOp::I64x2Splat): + CHECK(iter.readConversion(ValType::I64, ValType::V128, ¬hing)); + case uint32_t(SimdOp::F32x4Splat): + CHECK(iter.readConversion(ValType::F32, ValType::V128, ¬hing)); + case uint32_t(SimdOp::F64x2Splat): + CHECK(iter.readConversion(ValType::F64, ValType::V128, ¬hing)); + + case uint32_t(SimdOp::V128AnyTrue): + case uint32_t(SimdOp::I8x16AllTrue): + case uint32_t(SimdOp::I16x8AllTrue): + case uint32_t(SimdOp::I32x4AllTrue): + case uint32_t(SimdOp::I64x2AllTrue): + case uint32_t(SimdOp::I8x16Bitmask): + case uint32_t(SimdOp::I16x8Bitmask): + case uint32_t(SimdOp::I32x4Bitmask): + case uint32_t(SimdOp::I64x2Bitmask): + CHECK(iter.readConversion(ValType::V128, ValType::I32, ¬hing)); + + case uint32_t(SimdOp::I8x16ReplaceLane): + CHECK(iter.readReplaceLane(ValType::I32, 16, &noIndex, ¬hing, + ¬hing)); + case uint32_t(SimdOp::I16x8ReplaceLane): + CHECK(iter.readReplaceLane(ValType::I32, 8, &noIndex, ¬hing, + ¬hing)); + case uint32_t(SimdOp::I32x4ReplaceLane): + CHECK(iter.readReplaceLane(ValType::I32, 4, &noIndex, ¬hing, + ¬hing)); + case uint32_t(SimdOp::I64x2ReplaceLane): + CHECK(iter.readReplaceLane(ValType::I64, 2, &noIndex, ¬hing, + ¬hing)); + case uint32_t(SimdOp::F32x4ReplaceLane): + CHECK(iter.readReplaceLane(ValType::F32, 4, &noIndex, ¬hing, + ¬hing)); + case uint32_t(SimdOp::F64x2ReplaceLane): + CHECK(iter.readReplaceLane(ValType::F64, 2, &noIndex, ¬hing, + ¬hing)); + + case uint32_t(SimdOp::I8x16Eq): + case uint32_t(SimdOp::I8x16Ne): + case uint32_t(SimdOp::I8x16LtS): + case uint32_t(SimdOp::I8x16LtU): + case uint32_t(SimdOp::I8x16GtS): + case uint32_t(SimdOp::I8x16GtU): + case uint32_t(SimdOp::I8x16LeS): + case uint32_t(SimdOp::I8x16LeU): + case uint32_t(SimdOp::I8x16GeS): + case uint32_t(SimdOp::I8x16GeU): + case uint32_t(SimdOp::I16x8Eq): + case uint32_t(SimdOp::I16x8Ne): + case uint32_t(SimdOp::I16x8LtS): + case uint32_t(SimdOp::I16x8LtU): + case uint32_t(SimdOp::I16x8GtS): + case uint32_t(SimdOp::I16x8GtU): + case uint32_t(SimdOp::I16x8LeS): + case uint32_t(SimdOp::I16x8LeU): + case uint32_t(SimdOp::I16x8GeS): + case uint32_t(SimdOp::I16x8GeU): + case uint32_t(SimdOp::I32x4Eq): + case uint32_t(SimdOp::I32x4Ne): + case uint32_t(SimdOp::I32x4LtS): + case uint32_t(SimdOp::I32x4LtU): + case uint32_t(SimdOp::I32x4GtS): + case uint32_t(SimdOp::I32x4GtU): + case uint32_t(SimdOp::I32x4LeS): + case uint32_t(SimdOp::I32x4LeU): + case uint32_t(SimdOp::I32x4GeS): + case uint32_t(SimdOp::I32x4GeU): + case uint32_t(SimdOp::I64x2Eq): + case uint32_t(SimdOp::I64x2Ne): + case uint32_t(SimdOp::I64x2LtS): + case uint32_t(SimdOp::I64x2GtS): + case uint32_t(SimdOp::I64x2LeS): + case uint32_t(SimdOp::I64x2GeS): + case uint32_t(SimdOp::F32x4Eq): + case uint32_t(SimdOp::F32x4Ne): + case uint32_t(SimdOp::F32x4Lt): + case uint32_t(SimdOp::F32x4Gt): + case uint32_t(SimdOp::F32x4Le): + case uint32_t(SimdOp::F32x4Ge): + case uint32_t(SimdOp::F64x2Eq): + case uint32_t(SimdOp::F64x2Ne): + case uint32_t(SimdOp::F64x2Lt): + case uint32_t(SimdOp::F64x2Gt): + case uint32_t(SimdOp::F64x2Le): + case uint32_t(SimdOp::F64x2Ge): + case uint32_t(SimdOp::V128And): + case uint32_t(SimdOp::V128Or): + case uint32_t(SimdOp::V128Xor): + case uint32_t(SimdOp::V128AndNot): + case uint32_t(SimdOp::I8x16AvgrU): + case uint32_t(SimdOp::I16x8AvgrU): + case uint32_t(SimdOp::I8x16Add): + case uint32_t(SimdOp::I8x16AddSatS): + case uint32_t(SimdOp::I8x16AddSatU): + case uint32_t(SimdOp::I8x16Sub): + case uint32_t(SimdOp::I8x16SubSatS): + case uint32_t(SimdOp::I8x16SubSatU): + case uint32_t(SimdOp::I8x16MinS): + case uint32_t(SimdOp::I8x16MinU): + case uint32_t(SimdOp::I8x16MaxS): + case uint32_t(SimdOp::I8x16MaxU): + case uint32_t(SimdOp::I16x8Add): + case uint32_t(SimdOp::I16x8AddSatS): + case uint32_t(SimdOp::I16x8AddSatU): + case uint32_t(SimdOp::I16x8Sub): + case uint32_t(SimdOp::I16x8SubSatS): + case uint32_t(SimdOp::I16x8SubSatU): + case uint32_t(SimdOp::I16x8Mul): + case uint32_t(SimdOp::I16x8MinS): + case uint32_t(SimdOp::I16x8MinU): + case uint32_t(SimdOp::I16x8MaxS): + case uint32_t(SimdOp::I16x8MaxU): + case uint32_t(SimdOp::I32x4Add): + case uint32_t(SimdOp::I32x4Sub): + case uint32_t(SimdOp::I32x4Mul): + case uint32_t(SimdOp::I32x4MinS): + case uint32_t(SimdOp::I32x4MinU): + case uint32_t(SimdOp::I32x4MaxS): + case uint32_t(SimdOp::I32x4MaxU): + case uint32_t(SimdOp::I64x2Add): + case uint32_t(SimdOp::I64x2Sub): + case uint32_t(SimdOp::I64x2Mul): + case uint32_t(SimdOp::F32x4Add): + case uint32_t(SimdOp::F32x4Sub): + case uint32_t(SimdOp::F32x4Mul): + case uint32_t(SimdOp::F32x4Div): + case uint32_t(SimdOp::F32x4Min): + case uint32_t(SimdOp::F32x4Max): + case uint32_t(SimdOp::F64x2Add): + case uint32_t(SimdOp::F64x2Sub): + case uint32_t(SimdOp::F64x2Mul): + case uint32_t(SimdOp::F64x2Div): + case uint32_t(SimdOp::F64x2Min): + case uint32_t(SimdOp::F64x2Max): + case uint32_t(SimdOp::I8x16NarrowI16x8S): + case uint32_t(SimdOp::I8x16NarrowI16x8U): + case uint32_t(SimdOp::I16x8NarrowI32x4S): + case uint32_t(SimdOp::I16x8NarrowI32x4U): + case uint32_t(SimdOp::I8x16Swizzle): + case uint32_t(SimdOp::F32x4PMax): + case uint32_t(SimdOp::F32x4PMin): + case uint32_t(SimdOp::F64x2PMax): + case uint32_t(SimdOp::F64x2PMin): + case uint32_t(SimdOp::I32x4DotI16x8S): + case uint32_t(SimdOp::I16x8ExtmulLowI8x16S): + case uint32_t(SimdOp::I16x8ExtmulHighI8x16S): + case uint32_t(SimdOp::I16x8ExtmulLowI8x16U): + case uint32_t(SimdOp::I16x8ExtmulHighI8x16U): + case uint32_t(SimdOp::I32x4ExtmulLowI16x8S): + case uint32_t(SimdOp::I32x4ExtmulHighI16x8S): + case uint32_t(SimdOp::I32x4ExtmulLowI16x8U): + case uint32_t(SimdOp::I32x4ExtmulHighI16x8U): + case uint32_t(SimdOp::I64x2ExtmulLowI32x4S): + case uint32_t(SimdOp::I64x2ExtmulHighI32x4S): + case uint32_t(SimdOp::I64x2ExtmulLowI32x4U): + case uint32_t(SimdOp::I64x2ExtmulHighI32x4U): + case uint32_t(SimdOp::I16x8Q15MulrSatS): + CHECK(iter.readBinary(ValType::V128, ¬hing, ¬hing)); + + case uint32_t(SimdOp::I8x16Neg): + case uint32_t(SimdOp::I16x8Neg): + case uint32_t(SimdOp::I16x8ExtendLowI8x16S): + case uint32_t(SimdOp::I16x8ExtendHighI8x16S): + case uint32_t(SimdOp::I16x8ExtendLowI8x16U): + case uint32_t(SimdOp::I16x8ExtendHighI8x16U): + case uint32_t(SimdOp::I32x4Neg): + case uint32_t(SimdOp::I32x4ExtendLowI16x8S): + case uint32_t(SimdOp::I32x4ExtendHighI16x8S): + case uint32_t(SimdOp::I32x4ExtendLowI16x8U): + case uint32_t(SimdOp::I32x4ExtendHighI16x8U): + case uint32_t(SimdOp::I32x4TruncSatF32x4S): + case uint32_t(SimdOp::I32x4TruncSatF32x4U): + case uint32_t(SimdOp::I64x2Neg): + case uint32_t(SimdOp::I64x2ExtendLowI32x4S): + case uint32_t(SimdOp::I64x2ExtendHighI32x4S): + case uint32_t(SimdOp::I64x2ExtendLowI32x4U): + case uint32_t(SimdOp::I64x2ExtendHighI32x4U): + case uint32_t(SimdOp::F32x4Abs): + case uint32_t(SimdOp::F32x4Neg): + case uint32_t(SimdOp::F32x4Sqrt): + case uint32_t(SimdOp::F32x4ConvertI32x4S): + case uint32_t(SimdOp::F32x4ConvertI32x4U): + case uint32_t(SimdOp::F64x2Abs): + case uint32_t(SimdOp::F64x2Neg): + case uint32_t(SimdOp::F64x2Sqrt): + case uint32_t(SimdOp::V128Not): + case uint32_t(SimdOp::I8x16Popcnt): + case uint32_t(SimdOp::I8x16Abs): + case uint32_t(SimdOp::I16x8Abs): + case uint32_t(SimdOp::I32x4Abs): + case uint32_t(SimdOp::I64x2Abs): + case uint32_t(SimdOp::F32x4Ceil): + case uint32_t(SimdOp::F32x4Floor): + case uint32_t(SimdOp::F32x4Trunc): + case uint32_t(SimdOp::F32x4Nearest): + case uint32_t(SimdOp::F64x2Ceil): + case uint32_t(SimdOp::F64x2Floor): + case uint32_t(SimdOp::F64x2Trunc): + case uint32_t(SimdOp::F64x2Nearest): + case uint32_t(SimdOp::F32x4DemoteF64x2Zero): + case uint32_t(SimdOp::F64x2PromoteLowF32x4): + case uint32_t(SimdOp::F64x2ConvertLowI32x4S): + case uint32_t(SimdOp::F64x2ConvertLowI32x4U): + case uint32_t(SimdOp::I32x4TruncSatF64x2SZero): + case uint32_t(SimdOp::I32x4TruncSatF64x2UZero): + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16S): + case uint32_t(SimdOp::I16x8ExtaddPairwiseI8x16U): + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8S): + case uint32_t(SimdOp::I32x4ExtaddPairwiseI16x8U): + CHECK(iter.readUnary(ValType::V128, ¬hing)); + + case uint32_t(SimdOp::I8x16Shl): + case uint32_t(SimdOp::I8x16ShrS): + case uint32_t(SimdOp::I8x16ShrU): + case uint32_t(SimdOp::I16x8Shl): + case uint32_t(SimdOp::I16x8ShrS): + case uint32_t(SimdOp::I16x8ShrU): + case uint32_t(SimdOp::I32x4Shl): + case uint32_t(SimdOp::I32x4ShrS): + case uint32_t(SimdOp::I32x4ShrU): + case uint32_t(SimdOp::I64x2Shl): + case uint32_t(SimdOp::I64x2ShrS): + case uint32_t(SimdOp::I64x2ShrU): + CHECK(iter.readVectorShift(¬hing, ¬hing)); + + case uint32_t(SimdOp::V128Bitselect): + CHECK( + iter.readTernary(ValType::V128, ¬hing, ¬hing, ¬hing)); + + case uint32_t(SimdOp::I8x16Shuffle): { + V128 mask; + CHECK(iter.readVectorShuffle(¬hing, ¬hing, &mask)); + } + + case uint32_t(SimdOp::V128Const): { + V128 noVector; + CHECK(iter.readV128Const(&noVector)); + } + + case uint32_t(SimdOp::V128Load): { + LinearMemoryAddress addr; + CHECK(iter.readLoad(ValType::V128, 16, &addr)); + } + + case uint32_t(SimdOp::V128Load8Splat): { + LinearMemoryAddress addr; + CHECK(iter.readLoadSplat(1, &addr)); + } + + case uint32_t(SimdOp::V128Load16Splat): { + LinearMemoryAddress addr; + CHECK(iter.readLoadSplat(2, &addr)); + } + + case uint32_t(SimdOp::V128Load32Splat): { + LinearMemoryAddress addr; + CHECK(iter.readLoadSplat(4, &addr)); + } + + case uint32_t(SimdOp::V128Load64Splat): { + LinearMemoryAddress addr; + CHECK(iter.readLoadSplat(8, &addr)); + } + + case uint32_t(SimdOp::V128Load8x8S): + case uint32_t(SimdOp::V128Load8x8U): { + LinearMemoryAddress addr; + CHECK(iter.readLoadExtend(&addr)); + } + + case uint32_t(SimdOp::V128Load16x4S): + case uint32_t(SimdOp::V128Load16x4U): { + LinearMemoryAddress addr; + CHECK(iter.readLoadExtend(&addr)); + } + + case uint32_t(SimdOp::V128Load32x2S): + case uint32_t(SimdOp::V128Load32x2U): { + LinearMemoryAddress addr; + CHECK(iter.readLoadExtend(&addr)); + } + + case uint32_t(SimdOp::V128Store): { + LinearMemoryAddress addr; + CHECK(iter.readStore(ValType::V128, 16, &addr, ¬hing)); + } + + case uint32_t(SimdOp::V128Load32Zero): { + LinearMemoryAddress addr; + CHECK(iter.readLoadSplat(4, &addr)); + } + + case uint32_t(SimdOp::V128Load64Zero): { + LinearMemoryAddress addr; + CHECK(iter.readLoadSplat(8, &addr)); + } + + case uint32_t(SimdOp::V128Load8Lane): { + LinearMemoryAddress addr; + CHECK(iter.readLoadLane(1, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Load16Lane): { + LinearMemoryAddress addr; + CHECK(iter.readLoadLane(2, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Load32Lane): { + LinearMemoryAddress addr; + CHECK(iter.readLoadLane(4, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Load64Lane): { + LinearMemoryAddress addr; + CHECK(iter.readLoadLane(8, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Store8Lane): { + LinearMemoryAddress addr; + CHECK(iter.readStoreLane(1, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Store16Lane): { + LinearMemoryAddress addr; + CHECK(iter.readStoreLane(2, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Store32Lane): { + LinearMemoryAddress addr; + CHECK(iter.readStoreLane(4, &addr, &noIndex, ¬hing)); + } + + case uint32_t(SimdOp::V128Store64Lane): { + LinearMemoryAddress addr; + CHECK(iter.readStoreLane(8, &addr, &noIndex, ¬hing)); + } + +# ifdef ENABLE_WASM_RELAXED_SIMD + case uint32_t(SimdOp::F32x4RelaxedFma): + case uint32_t(SimdOp::F32x4RelaxedFnma): + case uint32_t(SimdOp::F64x2RelaxedFma): + case uint32_t(SimdOp::F64x2RelaxedFnma): + case uint32_t(SimdOp::I8x16RelaxedLaneSelect): + case uint32_t(SimdOp::I16x8RelaxedLaneSelect): + case uint32_t(SimdOp::I32x4RelaxedLaneSelect): + case uint32_t(SimdOp::I64x2RelaxedLaneSelect): + case uint32_t(SimdOp::I32x4DotI8x16I7x16AddS): { + if (!env.v128RelaxedEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK( + iter.readTernary(ValType::V128, ¬hing, ¬hing, ¬hing)); + } + case uint32_t(SimdOp::F32x4RelaxedMin): + case uint32_t(SimdOp::F32x4RelaxedMax): + case uint32_t(SimdOp::F64x2RelaxedMin): + case uint32_t(SimdOp::F64x2RelaxedMax): + case uint32_t(SimdOp::I16x8RelaxedQ15MulrS): + case uint32_t(SimdOp::I16x8DotI8x16I7x16S): { + if (!env.v128RelaxedEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readBinary(ValType::V128, ¬hing, ¬hing)); + } + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4S): + case uint32_t(SimdOp::I32x4RelaxedTruncF32x4U): + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2SZero): + case uint32_t(SimdOp::I32x4RelaxedTruncF64x2UZero): { + if (!env.v128RelaxedEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readUnary(ValType::V128, ¬hing)); + } + case uint32_t(SimdOp::I8x16RelaxedSwizzle): { + if (!env.v128RelaxedEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readBinary(ValType::V128, ¬hing, ¬hing)); + } +# endif + + default: + return iter.unrecognizedOpcode(&op); + } + break; + } +#endif // ENABLE_WASM_SIMD + + case uint16_t(Op::MiscPrefix): { + switch (op.b1) { + case uint32_t(MiscOp::I32TruncSatF32S): + case uint32_t(MiscOp::I32TruncSatF32U): + CHECK(iter.readConversion(ValType::F32, ValType::I32, ¬hing)); + case uint32_t(MiscOp::I32TruncSatF64S): + case uint32_t(MiscOp::I32TruncSatF64U): + CHECK(iter.readConversion(ValType::F64, ValType::I32, ¬hing)); + case uint32_t(MiscOp::I64TruncSatF32S): + case uint32_t(MiscOp::I64TruncSatF32U): + CHECK(iter.readConversion(ValType::F32, ValType::I64, ¬hing)); + case uint32_t(MiscOp::I64TruncSatF64S): + case uint32_t(MiscOp::I64TruncSatF64U): + CHECK(iter.readConversion(ValType::F64, ValType::I64, ¬hing)); + case uint32_t(MiscOp::MemoryCopy): { + uint32_t unusedDestMemIndex; + uint32_t unusedSrcMemIndex; + CHECK(iter.readMemOrTableCopy(/*isMem=*/true, &unusedDestMemIndex, + ¬hing, &unusedSrcMemIndex, + ¬hing, ¬hing)); + } + case uint32_t(MiscOp::DataDrop): { + uint32_t unusedSegIndex; + CHECK(iter.readDataOrElemDrop(/*isData=*/true, &unusedSegIndex)); + } + case uint32_t(MiscOp::MemoryFill): + CHECK(iter.readMemFill(¬hing, ¬hing, ¬hing)); + case uint32_t(MiscOp::MemoryInit): { + uint32_t unusedSegIndex; + uint32_t unusedTableIndex; + CHECK(iter.readMemOrTableInit(/*isMem=*/true, &unusedSegIndex, + &unusedTableIndex, ¬hing, ¬hing, + ¬hing)); + } + case uint32_t(MiscOp::TableCopy): { + uint32_t unusedDestTableIndex; + uint32_t unusedSrcTableIndex; + CHECK(iter.readMemOrTableCopy( + /*isMem=*/false, &unusedDestTableIndex, ¬hing, + &unusedSrcTableIndex, ¬hing, ¬hing)); + } + case uint32_t(MiscOp::ElemDrop): { + uint32_t unusedSegIndex; + CHECK(iter.readDataOrElemDrop(/*isData=*/false, &unusedSegIndex)); + } + case uint32_t(MiscOp::TableInit): { + uint32_t unusedSegIndex; + uint32_t unusedTableIndex; + CHECK(iter.readMemOrTableInit(/*isMem=*/false, &unusedSegIndex, + &unusedTableIndex, ¬hing, ¬hing, + ¬hing)); + } + case uint32_t(MiscOp::TableFill): { + uint32_t unusedTableIndex; + CHECK(iter.readTableFill(&unusedTableIndex, ¬hing, ¬hing, + ¬hing)); + } +#ifdef ENABLE_WASM_MEMORY_CONTROL + case uint32_t(MiscOp::MemoryDiscard): { + if (!env.memoryControlEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readMemDiscard(¬hing, ¬hing)); + } +#endif + case uint32_t(MiscOp::TableGrow): { + uint32_t unusedTableIndex; + CHECK(iter.readTableGrow(&unusedTableIndex, ¬hing, ¬hing)); + } + case uint32_t(MiscOp::TableSize): { + uint32_t unusedTableIndex; + CHECK(iter.readTableSize(&unusedTableIndex)); + } + default: + return iter.unrecognizedOpcode(&op); + } + break; + } +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + case uint16_t(Op::RefAsNonNull): { + if (!env.functionReferencesEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readRefAsNonNull(¬hing)); + } + case uint16_t(Op::BrOnNull): { + if (!env.functionReferencesEnabled()) { + return iter.unrecognizedOpcode(&op); + } + uint32_t unusedDepth; + CHECK( + iter.readBrOnNull(&unusedDepth, &unusedType, ¬hings, ¬hing)); + } + case uint16_t(Op::BrOnNonNull): { + if (!env.functionReferencesEnabled()) { + return iter.unrecognizedOpcode(&op); + } + uint32_t unusedDepth; + CHECK(iter.readBrOnNonNull(&unusedDepth, &unusedType, ¬hings, + ¬hing)); + } +#endif +#ifdef ENABLE_WASM_GC + case uint16_t(Op::RefEq): { + if (!env.gcEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readComparison(RefType::eq(), ¬hing, ¬hing)); + } +#endif + case uint16_t(Op::RefFunc): { + uint32_t unusedIndex; + CHECK(iter.readRefFunc(&unusedIndex)); + } + case uint16_t(Op::RefNull): { + RefType type; + CHECK(iter.readRefNull(&type)); + } + case uint16_t(Op::RefIsNull): { + Nothing nothing; + CHECK(iter.readRefIsNull(¬hing)); + } + case uint16_t(Op::Try): + if (!env.exceptionsEnabled()) { + return iter.unrecognizedOpcode(&op); + } + CHECK(iter.readTry(&unusedType)); + case uint16_t(Op::Catch): { + if (!env.exceptionsEnabled()) { + return iter.unrecognizedOpcode(&op); + } + LabelKind unusedKind; + uint32_t unusedIndex; + CHECK(iter.readCatch(&unusedKind, &unusedIndex, &unusedType, + &unusedType, ¬hings)); + } + case uint16_t(Op::CatchAll): { + if (!env.exceptionsEnabled()) { + return iter.unrecognizedOpcode(&op); + } + LabelKind unusedKind; + CHECK(iter.readCatchAll(&unusedKind, &unusedType, &unusedType, + ¬hings)); + } + case uint16_t(Op::Delegate): { + if (!env.exceptionsEnabled()) { + return iter.unrecognizedOpcode(&op); + } + uint32_t unusedDepth; + if (!iter.readDelegate(&unusedDepth, &unusedType, ¬hings)) { + return false; + } + iter.popDelegate(); + break; + } + case uint16_t(Op::Throw): { + if (!env.exceptionsEnabled()) { + return iter.unrecognizedOpcode(&op); + } + uint32_t unusedIndex; + CHECK(iter.readThrow(&unusedIndex, ¬hings)); + } + case uint16_t(Op::Rethrow): { + if (!env.exceptionsEnabled()) { + return iter.unrecognizedOpcode(&op); + } + uint32_t unusedDepth; + CHECK(iter.readRethrow(&unusedDepth)); + } + case uint16_t(Op::ThreadPrefix): { + // Though thread ops can be used on nonshared memories, we make them + // unavailable if shared memory has been disabled in the prefs, for + // maximum predictability and safety and consistency with JS. + if (env.sharedMemoryEnabled() == Shareable::False) { + return iter.unrecognizedOpcode(&op); + } + switch (op.b1) { + case uint32_t(ThreadOp::Wake): { + LinearMemoryAddress addr; + CHECK(iter.readWake(&addr, ¬hing)); + } + case uint32_t(ThreadOp::I32Wait): { + LinearMemoryAddress addr; + CHECK(iter.readWait(&addr, ValType::I32, 4, ¬hing, ¬hing)); + } + case uint32_t(ThreadOp::I64Wait): { + LinearMemoryAddress addr; + CHECK(iter.readWait(&addr, ValType::I64, 8, ¬hing, ¬hing)); + } + case uint32_t(ThreadOp::Fence): { + CHECK(iter.readFence()); + } + case uint32_t(ThreadOp::I32AtomicLoad): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I32, 4)); + } + case uint32_t(ThreadOp::I64AtomicLoad): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I64, 8)); + } + case uint32_t(ThreadOp::I32AtomicLoad8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I32, 1)); + } + case uint32_t(ThreadOp::I32AtomicLoad16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I32, 2)); + } + case uint32_t(ThreadOp::I64AtomicLoad8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I64, 1)); + } + case uint32_t(ThreadOp::I64AtomicLoad16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I64, 2)); + } + case uint32_t(ThreadOp::I64AtomicLoad32U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicLoad(&addr, ValType::I64, 4)); + } + case uint32_t(ThreadOp::I32AtomicStore): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I32, 4, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicStore): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I64, 8, ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicStore8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I32, 1, ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicStore16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I32, 2, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicStore8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I64, 1, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicStore16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I64, 2, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicStore32U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicStore(&addr, ValType::I64, 4, ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicAdd): + case uint32_t(ThreadOp::I32AtomicSub): + case uint32_t(ThreadOp::I32AtomicAnd): + case uint32_t(ThreadOp::I32AtomicOr): + case uint32_t(ThreadOp::I32AtomicXor): + case uint32_t(ThreadOp::I32AtomicXchg): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I32, 4, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicAdd): + case uint32_t(ThreadOp::I64AtomicSub): + case uint32_t(ThreadOp::I64AtomicAnd): + case uint32_t(ThreadOp::I64AtomicOr): + case uint32_t(ThreadOp::I64AtomicXor): + case uint32_t(ThreadOp::I64AtomicXchg): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I64, 8, ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicAdd8U): + case uint32_t(ThreadOp::I32AtomicSub8U): + case uint32_t(ThreadOp::I32AtomicAnd8U): + case uint32_t(ThreadOp::I32AtomicOr8U): + case uint32_t(ThreadOp::I32AtomicXor8U): + case uint32_t(ThreadOp::I32AtomicXchg8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I32, 1, ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicAdd16U): + case uint32_t(ThreadOp::I32AtomicSub16U): + case uint32_t(ThreadOp::I32AtomicAnd16U): + case uint32_t(ThreadOp::I32AtomicOr16U): + case uint32_t(ThreadOp::I32AtomicXor16U): + case uint32_t(ThreadOp::I32AtomicXchg16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I32, 2, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicAdd8U): + case uint32_t(ThreadOp::I64AtomicSub8U): + case uint32_t(ThreadOp::I64AtomicAnd8U): + case uint32_t(ThreadOp::I64AtomicOr8U): + case uint32_t(ThreadOp::I64AtomicXor8U): + case uint32_t(ThreadOp::I64AtomicXchg8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I64, 1, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicAdd16U): + case uint32_t(ThreadOp::I64AtomicSub16U): + case uint32_t(ThreadOp::I64AtomicAnd16U): + case uint32_t(ThreadOp::I64AtomicOr16U): + case uint32_t(ThreadOp::I64AtomicXor16U): + case uint32_t(ThreadOp::I64AtomicXchg16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I64, 2, ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicAdd32U): + case uint32_t(ThreadOp::I64AtomicSub32U): + case uint32_t(ThreadOp::I64AtomicAnd32U): + case uint32_t(ThreadOp::I64AtomicOr32U): + case uint32_t(ThreadOp::I64AtomicXor32U): + case uint32_t(ThreadOp::I64AtomicXchg32U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicRMW(&addr, ValType::I64, 4, ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicCmpXchg): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I32, 4, ¬hing, + ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicCmpXchg): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I64, 8, ¬hing, + ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicCmpXchg8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I32, 1, ¬hing, + ¬hing)); + } + case uint32_t(ThreadOp::I32AtomicCmpXchg16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I32, 2, ¬hing, + ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicCmpXchg8U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I64, 1, ¬hing, + ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicCmpXchg16U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I64, 2, ¬hing, + ¬hing)); + } + case uint32_t(ThreadOp::I64AtomicCmpXchg32U): { + LinearMemoryAddress addr; + CHECK(iter.readAtomicCmpXchg(&addr, ValType::I64, 4, ¬hing, + ¬hing)); + } + default: + return iter.unrecognizedOpcode(&op); + } + break; + } + case uint16_t(Op::MozPrefix): + return iter.unrecognizedOpcode(&op); + default: + return iter.unrecognizedOpcode(&op); + } + } + + MOZ_CRASH("unreachable"); + +#undef CHECK +} + +bool wasm::ValidateFunctionBody(const ModuleEnvironment& env, + uint32_t funcIndex, uint32_t bodySize, + Decoder& d) { + ValTypeVector locals; + if (!locals.appendAll(env.funcs[funcIndex].type->args())) { + return false; + } + + const uint8_t* bodyBegin = d.currentPosition(); + + if (!DecodeLocalEntries(d, *env.types, env.features, &locals)) { + return false; + } + + if (!DecodeFunctionBodyExprs(env, funcIndex, locals, bodyBegin + bodySize, + &d)) { + return false; + } + + return true; +} + +// Section macros. + +static bool DecodePreamble(Decoder& d) { + if (d.bytesRemain() > MaxModuleBytes) { + return d.fail("module too big"); + } + + uint32_t u32; + if (!d.readFixedU32(&u32) || u32 != MagicNumber) { + return d.fail("failed to match magic number"); + } + + if (!d.readFixedU32(&u32) || u32 != EncodingVersion) { + return d.failf("binary version 0x%" PRIx32 + " does not match expected version 0x%" PRIx32, + u32, EncodingVersion); + } + + return true; +} + +static bool DecodeValTypeVector(Decoder& d, ModuleEnvironment* env, + uint32_t count, ValTypeVector* valTypes) { + if (!valTypes->resize(count)) { + return false; + } + + for (uint32_t i = 0; i < count; i++) { + if (!d.readValType(*env->types, env->features, &(*valTypes)[i])) { + return false; + } + } + return true; +} + +static bool DecodeFuncType(Decoder& d, ModuleEnvironment* env, + FuncType* funcType) { + uint32_t numArgs; + if (!d.readVarU32(&numArgs)) { + return d.fail("bad number of function args"); + } + if (numArgs > MaxParams) { + return d.fail("too many arguments in signature"); + } + ValTypeVector args; + if (!DecodeValTypeVector(d, env, numArgs, &args)) { + return false; + } + + uint32_t numResults; + if (!d.readVarU32(&numResults)) { + return d.fail("bad number of function returns"); + } + if (numResults > MaxResults) { + return d.fail("too many returns in signature"); + } + ValTypeVector results; + if (!DecodeValTypeVector(d, env, numResults, &results)) { + return false; + } + + *funcType = FuncType(std::move(args), std::move(results)); + return true; +} + +static bool DecodeStructType(Decoder& d, ModuleEnvironment* env, + StructType* structType) { + if (!env->gcEnabled()) { + return d.fail("Structure types not enabled"); + } + + uint32_t numFields; + if (!d.readVarU32(&numFields)) { + return d.fail("Bad number of fields"); + } + + if (numFields > MaxStructFields) { + return d.fail("too many fields in struct"); + } + + StructFieldVector fields; + if (!fields.resize(numFields)) { + return false; + } + + for (uint32_t i = 0; i < numFields; i++) { + if (!d.readFieldType(*env->types, env->features, &fields[i].type)) { + return false; + } + + uint8_t flags; + if (!d.readFixedU8(&flags)) { + return d.fail("expected flag"); + } + if ((flags & ~uint8_t(FieldFlags::AllowedMask)) != 0) { + return d.fail("garbage flag bits"); + } + fields[i].isMutable = flags & uint8_t(FieldFlags::Mutable); + } + + *structType = StructType(std::move(fields)); + + // Compute the struct layout, and fail if the struct is too large + if (!structType->init()) { + return d.fail("too many fields in struct"); + } + return true; +} + +static bool DecodeArrayType(Decoder& d, ModuleEnvironment* env, + ArrayType* arrayType) { + if (!env->gcEnabled()) { + return d.fail("gc types not enabled"); + } + + FieldType elementType; + if (!d.readFieldType(*env->types, env->features, &elementType)) { + return false; + } + + uint8_t flags; + if (!d.readFixedU8(&flags)) { + return d.fail("expected flag"); + } + if ((flags & ~uint8_t(FieldFlags::AllowedMask)) != 0) { + return d.fail("garbage flag bits"); + } + bool isMutable = flags & uint8_t(FieldFlags::Mutable); + + *arrayType = ArrayType(elementType, isMutable); + return true; +} + +static bool DecodeTypeSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Type, env, &range, "type")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numRecGroups; + if (!d.readVarU32(&numRecGroups)) { + return d.fail("expected number of types"); + } + + // Check if we've reached our implementation defined limit of recursion + // groups. + if (numRecGroups > MaxRecGroups) { + return d.fail("too many types"); + } + + for (uint32_t recGroupIndex = 0; recGroupIndex < numRecGroups; + recGroupIndex++) { + uint32_t recGroupLength = 1; + + // Decode an optional recursion group length, if the GC proposal is + // enabled. + if (env->gcEnabled()) { + uint8_t firstTypeCode; + if (!d.peekByte(&firstTypeCode)) { + return d.fail("expected type form"); + } + + if (firstTypeCode == (uint8_t)TypeCode::RecGroup || + firstTypeCode == (uint8_t)TypeCode::RecGroupOld) { + // Skip over the prefix byte that was peeked. + d.uncheckedReadFixedU8(); + + // Read the number of types in this recursion group + if (!d.readVarU32(&recGroupLength)) { + return d.fail("expected recursion group length"); + } + } + } + + // Start a recursion group. This will extend the type context with empty + // type definitions to be filled. + MutableRecGroup recGroup = env->types->startRecGroup(recGroupLength); + if (!recGroup) { + return false; + } + + // First, iterate over the types, validate them and set super types. + // Subtyping relationship will be checked in a second iteration. + for (uint32_t recGroupTypeIndex = 0; recGroupTypeIndex < recGroupLength; + recGroupTypeIndex++) { + uint32_t typeIndex = + env->types->length() - recGroupLength + recGroupTypeIndex; + + // Check if we've reached our implementation defined limit of type + // definitions. + if (typeIndex > MaxTypes) { + return d.fail("too many types"); + } + + uint8_t form; + const TypeDef* superTypeDef = nullptr; + + // Decode an optional declared super type index, if the GC proposal is + // enabled. + if (env->gcEnabled() && d.peekByte(&form) && + form == (uint8_t)TypeCode::SubType) { + // Skip over the `sub` prefix byte we peeked. + d.uncheckedReadFixedU8(); + + // Decode the number of super types, which is currently limited to at + // most one. + uint32_t numSuperTypes; + if (!d.readVarU32(&numSuperTypes)) { + return d.fail("expected number of super types"); + } + if (numSuperTypes > 1) { + return d.fail("too many super types"); + } + + // Decode the super type, if any. + if (numSuperTypes == 1) { + uint32_t superTypeDefIndex; + if (!d.readVarU32(&superTypeDefIndex)) { + return d.fail("expected super type index"); + } + + // A super type index must be strictly less than the current type + // index in order to avoid cycles. + if (superTypeDefIndex >= typeIndex) { + return d.fail("invalid super type index"); + } + + superTypeDef = &env->types->type(superTypeDefIndex); + } + } + + // Decode the kind of type definition + if (!d.readFixedU8(&form)) { + return d.fail("expected type form"); + } + + TypeDef* typeDef = &recGroup->type(recGroupTypeIndex); + switch (form) { + case uint8_t(TypeCode::Func): { + FuncType funcType; + if (!DecodeFuncType(d, env, &funcType)) { + return false; + } + *typeDef = std::move(funcType); + break; + } + case uint8_t(TypeCode::Struct): { + StructType structType; + if (!DecodeStructType(d, env, &structType)) { + return false; + } + *typeDef = std::move(structType); + break; + } + case uint8_t(TypeCode::Array): { + ArrayType arrayType; + if (!DecodeArrayType(d, env, &arrayType)) { + return false; + } + *typeDef = std::move(arrayType); + break; + } + default: + return d.fail("expected type form"); + } + + if (superTypeDef) { + // Check that we aren't creating too deep of a subtyping chain + if (superTypeDef->subTypingDepth() >= MaxSubTypingDepth) { + return d.fail("type is too deep"); + } + + typeDef->setSuperTypeDef(superTypeDef); + } + } + + // Check the super types to make sure they are compatible with their + // subtypes. This is done in a second iteration to avoid dealing with not + // yet loaded types. + for (uint32_t recGroupTypeIndex = 0; recGroupTypeIndex < recGroupLength; + recGroupTypeIndex++) { + TypeDef* typeDef = &recGroup->type(recGroupTypeIndex); + if (typeDef->superTypeDef()) { + // Check that the super type is compatible with this type + if (!TypeDef::canBeSubTypeOf(typeDef, typeDef->superTypeDef())) { + return d.fail("incompatible super type"); + } + } + } + + // Finish the recursion group, which will canonicalize the types. + if (!env->types->endRecGroup()) { + return false; + } + } + + return d.finishSection(*range, "type"); +} + +[[nodiscard]] static bool DecodeName(Decoder& d, CacheableName* name) { + uint32_t numBytes; + if (!d.readVarU32(&numBytes)) { + return false; + } + + if (numBytes > MaxStringBytes) { + return false; + } + + const uint8_t* bytes; + if (!d.readBytes(numBytes, &bytes)) { + return false; + } + + if (!IsUtf8(AsChars(Span(bytes, numBytes)))) { + return false; + } + + UTF8Bytes utf8Bytes; + if (!utf8Bytes.resizeUninitialized(numBytes)) { + return false; + } + memcpy(utf8Bytes.begin(), bytes, numBytes); + + *name = CacheableName(std::move(utf8Bytes)); + return true; +} + +static bool DecodeFuncTypeIndex(Decoder& d, const SharedTypeContext& types, + uint32_t* funcTypeIndex) { + if (!d.readVarU32(funcTypeIndex)) { + return d.fail("expected signature index"); + } + + if (*funcTypeIndex >= types->length()) { + return d.fail("signature index out of range"); + } + + const TypeDef& def = (*types)[*funcTypeIndex]; + + if (!def.isFuncType()) { + return d.fail("signature index references non-signature"); + } + + return true; +} + +static bool DecodeLimits(Decoder& d, LimitsKind kind, Limits* limits) { + uint8_t flags; + if (!d.readFixedU8(&flags)) { + return d.fail("expected flags"); + } + + uint8_t mask = kind == LimitsKind::Memory ? uint8_t(LimitsMask::Memory) + : uint8_t(LimitsMask::Table); + + if (flags & ~uint8_t(mask)) { + return d.failf("unexpected bits set in flags: %" PRIu32, + uint32_t(flags & ~uint8_t(mask))); + } + + uint64_t initial; + if (!d.readVarU64(&initial)) { + return d.fail("expected initial length"); + } + limits->initial = initial; + + if (flags & uint8_t(LimitsFlags::HasMaximum)) { + uint64_t maximum; + if (!d.readVarU64(&maximum)) { + return d.fail("expected maximum length"); + } + + if (limits->initial > maximum) { + return d.failf( + "memory size minimum must not be greater than maximum; " + "maximum length %" PRIu64 " is less than initial length %" PRIu64, + maximum, limits->initial); + } + + limits->maximum.emplace(maximum); + } + + limits->shared = Shareable::False; + limits->indexType = IndexType::I32; + + // Memory limits may be shared or specify an alternate index type + if (kind == LimitsKind::Memory) { + if ((flags & uint8_t(LimitsFlags::IsShared)) && + !(flags & uint8_t(LimitsFlags::HasMaximum))) { + return d.fail("maximum length required for shared memory"); + } + + limits->shared = (flags & uint8_t(LimitsFlags::IsShared)) + ? Shareable::True + : Shareable::False; + +#ifdef ENABLE_WASM_MEMORY64 + limits->indexType = + (flags & uint8_t(LimitsFlags::IsI64)) ? IndexType::I64 : IndexType::I32; +#else + if (flags & uint8_t(LimitsFlags::IsI64)) { + return d.fail("i64 is not supported for memory limits"); + } +#endif + } + + return true; +} + +static bool DecodeTableTypeAndLimits(Decoder& d, ModuleEnvironment* env) { + bool initExprPresent = false; + uint8_t typeCode; + if (!d.peekByte(&typeCode)) { + return d.fail("expected type code"); + } + if (typeCode == (uint8_t)TypeCode::TableHasInitExpr) { + d.uncheckedReadFixedU8(); + uint8_t flags; + if (!d.readFixedU8(&flags) || flags != 0) { + return d.fail("expected reserved byte to be 0"); + } + initExprPresent = true; + } + + RefType tableElemType; + if (!d.readRefType(*env->types, env->features, &tableElemType)) { + return false; + } + + Limits limits; + if (!DecodeLimits(d, LimitsKind::Table, &limits)) { + return false; + } + + // Decoding limits for a table only supports i32 + MOZ_ASSERT(limits.indexType == IndexType::I32); + + // If there's a maximum, check it is in range. The check to exclude + // initial > maximum is carried out by the DecodeLimits call above, so + // we don't repeat it here. + if (limits.initial > MaxTableLimitField || + ((limits.maximum.isSome() && + limits.maximum.value() > MaxTableLimitField))) { + return d.fail("too many table elements"); + } + + if (env->tables.length() >= MaxTables) { + return d.fail("too many tables"); + } + + // The rest of the runtime expects table limits to be within a 32-bit range. + static_assert(MaxTableLimitField <= UINT32_MAX, "invariant"); + uint32_t initialLength = uint32_t(limits.initial); + Maybe maximumLength; + if (limits.maximum) { + maximumLength = Some(uint32_t(*limits.maximum)); + } + + Maybe initExpr; + if (initExprPresent) { + InitExpr initializer; + if (!InitExpr::decodeAndValidate(d, env, tableElemType, + env->globals.length(), &initializer)) { + return false; + } + initExpr = Some(std::move(initializer)); + } else { + if (!tableElemType.isNullable()) { + return d.fail("table with non-nullable references requires initializer"); + } + } + + return env->tables.emplaceBack(tableElemType, initialLength, maximumLength, + std::move(initExpr), /* isAsmJS */ false); +} + +static bool DecodeGlobalType(Decoder& d, const SharedTypeContext& types, + const FeatureArgs& features, ValType* type, + bool* isMutable) { + if (!d.readValType(*types, features, type)) { + return d.fail("expected global type"); + } + + uint8_t flags; + if (!d.readFixedU8(&flags)) { + return d.fail("expected global flags"); + } + + if (flags & ~uint8_t(GlobalTypeImmediate::AllowedMask)) { + return d.fail("unexpected bits set in global flags"); + } + + *isMutable = flags & uint8_t(GlobalTypeImmediate::IsMutable); + return true; +} + +static bool DecodeMemoryTypeAndLimits(Decoder& d, ModuleEnvironment* env) { + if (env->usesMemory()) { + return d.fail("already have default memory"); + } + + Limits limits; + if (!DecodeLimits(d, LimitsKind::Memory, &limits)) { + return false; + } + + uint64_t maxField = MaxMemoryLimitField(limits.indexType); + + if (limits.initial > maxField) { + return d.fail("initial memory size too big"); + } + + if (limits.maximum && *limits.maximum > maxField) { + return d.fail("maximum memory size too big"); + } + + if (limits.shared == Shareable::True && + env->sharedMemoryEnabled() == Shareable::False) { + return d.fail("shared memory is disabled"); + } + + if (limits.indexType == IndexType::I64 && !env->memory64Enabled()) { + return d.fail("memory64 is disabled"); + } + + env->memory = Some(MemoryDesc(limits)); + return true; +} + +static bool DecodeTag(Decoder& d, ModuleEnvironment* env, TagKind* tagKind, + uint32_t* funcTypeIndex) { + uint32_t tagCode; + if (!d.readVarU32(&tagCode)) { + return d.fail("expected tag kind"); + } + + if (TagKind(tagCode) != TagKind::Exception) { + return d.fail("illegal tag kind"); + } + *tagKind = TagKind(tagCode); + + if (!d.readVarU32(funcTypeIndex)) { + return d.fail("expected function index in tag"); + } + if (*funcTypeIndex >= env->numTypes()) { + return d.fail("function type index in tag out of bounds"); + } + if (!(*env->types)[*funcTypeIndex].isFuncType()) { + return d.fail("function type index must index a function type"); + } + if ((*env->types)[*funcTypeIndex].funcType().results().length() != 0) { + return d.fail("tag function types must not return anything"); + } + return true; +} + +static bool DecodeImport(Decoder& d, ModuleEnvironment* env) { + CacheableName moduleName; + if (!DecodeName(d, &moduleName)) { + return d.fail("expected valid import module name"); + } + + CacheableName funcName; + if (!DecodeName(d, &funcName)) { + return d.fail("expected valid import field name"); + } + + uint8_t rawImportKind; + if (!d.readFixedU8(&rawImportKind)) { + return d.fail("failed to read import kind"); + } + + DefinitionKind importKind = DefinitionKind(rawImportKind); + + switch (importKind) { + case DefinitionKind::Function: { + uint32_t funcTypeIndex; + if (!DecodeFuncTypeIndex(d, env->types, &funcTypeIndex)) { + return false; + } + if (!env->funcs.append(FuncDesc( + &env->types->type(funcTypeIndex).funcType(), funcTypeIndex))) { + return false; + } + if (env->funcs.length() > MaxFuncs) { + return d.fail("too many functions"); + } + break; + } + case DefinitionKind::Table: { + if (!DecodeTableTypeAndLimits(d, env)) { + return false; + } + env->tables.back().isImported = true; + break; + } + case DefinitionKind::Memory: { + if (!DecodeMemoryTypeAndLimits(d, env)) { + return false; + } + break; + } + case DefinitionKind::Global: { + ValType type; + bool isMutable; + if (!DecodeGlobalType(d, env->types, env->features, &type, &isMutable)) { + return false; + } + if (!env->globals.append( + GlobalDesc(type, isMutable, env->globals.length()))) { + return false; + } + if (env->globals.length() > MaxGlobals) { + return d.fail("too many globals"); + } + break; + } + case DefinitionKind::Tag: { + TagKind tagKind; + uint32_t funcTypeIndex; + if (!DecodeTag(d, env, &tagKind, &funcTypeIndex)) { + return false; + } + ValTypeVector args; + if (!args.appendAll((*env->types)[funcTypeIndex].funcType().args())) { + return false; + } + MutableTagType tagType = js_new(); + if (!tagType || !tagType->initialize(std::move(args))) { + return false; + } + if (!env->tags.emplaceBack(tagKind, tagType)) { + return false; + } + if (env->tags.length() > MaxTags) { + return d.fail("too many tags"); + } + break; + } + default: + return d.fail("unsupported import kind"); + } + + return env->imports.emplaceBack(std::move(moduleName), std::move(funcName), + importKind); +} + +static bool DecodeImportSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Import, env, &range, "import")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numImports; + if (!d.readVarU32(&numImports)) { + return d.fail("failed to read number of imports"); + } + + if (numImports > MaxImports) { + return d.fail("too many imports"); + } + + for (uint32_t i = 0; i < numImports; i++) { + if (!DecodeImport(d, env)) { + return false; + } + } + + if (!d.finishSection(*range, "import")) { + return false; + } + + env->numFuncImports = env->funcs.length(); + return true; +} + +static bool DecodeFunctionSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Function, env, &range, "function")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numDefs; + if (!d.readVarU32(&numDefs)) { + return d.fail("expected number of function definitions"); + } + + CheckedInt numFuncs = env->funcs.length(); + numFuncs += numDefs; + if (!numFuncs.isValid() || numFuncs.value() > MaxFuncs) { + return d.fail("too many functions"); + } + + if (!env->funcs.reserve(numFuncs.value())) { + return false; + } + + for (uint32_t i = 0; i < numDefs; i++) { + uint32_t funcTypeIndex; + if (!DecodeFuncTypeIndex(d, env->types, &funcTypeIndex)) { + return false; + } + env->funcs.infallibleAppend( + FuncDesc(&env->types->type(funcTypeIndex).funcType(), funcTypeIndex)); + } + + return d.finishSection(*range, "function"); +} + +static bool DecodeTableSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Table, env, &range, "table")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numTables; + if (!d.readVarU32(&numTables)) { + return d.fail("failed to read number of tables"); + } + + for (uint32_t i = 0; i < numTables; ++i) { + if (!DecodeTableTypeAndLimits(d, env)) { + return false; + } + } + + return d.finishSection(*range, "table"); +} + +static bool DecodeMemorySection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Memory, env, &range, "memory")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numMemories; + if (!d.readVarU32(&numMemories)) { + return d.fail("failed to read number of memories"); + } + + if (numMemories > 1) { + return d.fail("the number of memories must be at most one"); + } + + for (uint32_t i = 0; i < numMemories; ++i) { + if (!DecodeMemoryTypeAndLimits(d, env)) { + return false; + } + } + + return d.finishSection(*range, "memory"); +} + +static bool DecodeGlobalSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Global, env, &range, "global")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numDefs; + if (!d.readVarU32(&numDefs)) { + return d.fail("expected number of globals"); + } + + CheckedInt numGlobals = env->globals.length(); + numGlobals += numDefs; + if (!numGlobals.isValid() || numGlobals.value() > MaxGlobals) { + return d.fail("too many globals"); + } + + if (!env->globals.reserve(numGlobals.value())) { + return false; + } + + for (uint32_t i = 0; i < numDefs; i++) { + ValType type; + bool isMutable; + if (!DecodeGlobalType(d, env->types, env->features, &type, &isMutable)) { + return false; + } + + InitExpr initializer; + if (!InitExpr::decodeAndValidate(d, env, type, i, &initializer)) { + return false; + } + + env->globals.infallibleAppend( + GlobalDesc(std::move(initializer), isMutable)); + } + + return d.finishSection(*range, "global"); +} + +static bool DecodeTagSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Tag, env, &range, "tag")) { + return false; + } + if (!range) { + return true; + } + + if (!env->exceptionsEnabled()) { + return d.fail("exceptions not enabled"); + } + + uint32_t numDefs; + if (!d.readVarU32(&numDefs)) { + return d.fail("expected number of tags"); + } + + CheckedInt numTags = env->tags.length(); + numTags += numDefs; + if (!numTags.isValid() || numTags.value() > MaxTags) { + return d.fail("too many tags"); + } + + if (!env->tags.reserve(numTags.value())) { + return false; + } + + for (uint32_t i = 0; i < numDefs; i++) { + TagKind tagKind; + uint32_t funcTypeIndex; + if (!DecodeTag(d, env, &tagKind, &funcTypeIndex)) { + return false; + } + ValTypeVector args; + if (!args.appendAll((*env->types)[funcTypeIndex].funcType().args())) { + return false; + } + MutableTagType tagType = js_new(); + if (!tagType || !tagType->initialize(std::move(args))) { + return false; + } + env->tags.infallibleEmplaceBack(tagKind, tagType); + } + + return d.finishSection(*range, "tag"); +} + +using NameSet = HashSet, NameHasher, SystemAllocPolicy>; + +[[nodiscard]] static bool DecodeExportName(Decoder& d, NameSet* dupSet, + CacheableName* exportName) { + if (!DecodeName(d, exportName)) { + d.fail("expected valid export name"); + return false; + } + + NameSet::AddPtr p = dupSet->lookupForAdd(exportName->utf8Bytes()); + if (p) { + d.fail("duplicate export"); + return false; + } + + return dupSet->add(p, exportName->utf8Bytes()); +} + +static bool DecodeExport(Decoder& d, ModuleEnvironment* env, NameSet* dupSet) { + CacheableName fieldName; + if (!DecodeExportName(d, dupSet, &fieldName)) { + return false; + } + + uint8_t exportKind; + if (!d.readFixedU8(&exportKind)) { + return d.fail("failed to read export kind"); + } + + switch (DefinitionKind(exportKind)) { + case DefinitionKind::Function: { + uint32_t funcIndex; + if (!d.readVarU32(&funcIndex)) { + return d.fail("expected function index"); + } + + if (funcIndex >= env->numFuncs()) { + return d.fail("exported function index out of bounds"); + } + + env->declareFuncExported(funcIndex, /* eager */ true, + /* canRefFunc */ true); + return env->exports.emplaceBack(std::move(fieldName), funcIndex, + DefinitionKind::Function); + } + case DefinitionKind::Table: { + uint32_t tableIndex; + if (!d.readVarU32(&tableIndex)) { + return d.fail("expected table index"); + } + + if (tableIndex >= env->tables.length()) { + return d.fail("exported table index out of bounds"); + } + env->tables[tableIndex].isExported = true; + return env->exports.emplaceBack(std::move(fieldName), tableIndex, + DefinitionKind::Table); + } + case DefinitionKind::Memory: { + uint32_t memoryIndex; + if (!d.readVarU32(&memoryIndex)) { + return d.fail("expected memory index"); + } + + if (memoryIndex > 0 || !env->usesMemory()) { + return d.fail("exported memory index out of bounds"); + } + + return env->exports.emplaceBack(std::move(fieldName), + DefinitionKind::Memory); + } + case DefinitionKind::Global: { + uint32_t globalIndex; + if (!d.readVarU32(&globalIndex)) { + return d.fail("expected global index"); + } + + if (globalIndex >= env->globals.length()) { + return d.fail("exported global index out of bounds"); + } + + GlobalDesc* global = &env->globals[globalIndex]; + global->setIsExport(); + + return env->exports.emplaceBack(std::move(fieldName), globalIndex, + DefinitionKind::Global); + } + case DefinitionKind::Tag: { + uint32_t tagIndex; + if (!d.readVarU32(&tagIndex)) { + return d.fail("expected tag index"); + } + if (tagIndex >= env->tags.length()) { + return d.fail("exported tag index out of bounds"); + } + + env->tags[tagIndex].isExport = true; + return env->exports.emplaceBack(std::move(fieldName), tagIndex, + DefinitionKind::Tag); + } + default: + return d.fail("unexpected export kind"); + } + + MOZ_CRASH("unreachable"); +} + +static bool DecodeExportSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Export, env, &range, "export")) { + return false; + } + if (!range) { + return true; + } + + NameSet dupSet; + + uint32_t numExports; + if (!d.readVarU32(&numExports)) { + return d.fail("failed to read number of exports"); + } + + if (numExports > MaxExports) { + return d.fail("too many exports"); + } + + for (uint32_t i = 0; i < numExports; i++) { + if (!DecodeExport(d, env, &dupSet)) { + return false; + } + } + + return d.finishSection(*range, "export"); +} + +static bool DecodeStartSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Start, env, &range, "start")) { + return false; + } + if (!range) { + return true; + } + + uint32_t funcIndex; + if (!d.readVarU32(&funcIndex)) { + return d.fail("failed to read start func index"); + } + + if (funcIndex >= env->numFuncs()) { + return d.fail("unknown start function"); + } + + const FuncType& funcType = *env->funcs[funcIndex].type; + if (funcType.results().length() > 0) { + return d.fail("start function must not return anything"); + } + + if (funcType.args().length()) { + return d.fail("start function must be nullary"); + } + + env->declareFuncExported(funcIndex, /* eager */ true, /* canFuncRef */ false); + env->startFuncIndex = Some(funcIndex); + + return d.finishSection(*range, "start"); +} + +static inline ElemSegment::Kind NormalizeElemSegmentKind( + ElemSegmentKind decodedKind) { + switch (decodedKind) { + case ElemSegmentKind::Active: + case ElemSegmentKind::ActiveWithTableIndex: { + return ElemSegment::Kind::Active; + } + case ElemSegmentKind::Passive: { + return ElemSegment::Kind::Passive; + } + case ElemSegmentKind::Declared: { + return ElemSegment::Kind::Declared; + } + } + MOZ_CRASH("unexpected elem segment kind"); +} + +static bool DecodeElemSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Elem, env, &range, "elem")) { + return false; + } + if (!range) { + return true; + } + + uint32_t numSegments; + if (!d.readVarU32(&numSegments)) { + return d.fail("failed to read number of elem segments"); + } + + if (numSegments > MaxElemSegments) { + return d.fail("too many elem segments"); + } + + if (!env->elemSegments.reserve(numSegments)) { + return false; + } + + for (uint32_t i = 0; i < numSegments; i++) { + uint32_t segmentFlags; + if (!d.readVarU32(&segmentFlags)) { + return d.fail("expected elem segment flags field"); + } + + Maybe flags = ElemSegmentFlags::construct(segmentFlags); + if (!flags) { + return d.fail("invalid elem segment flags field"); + } + + MutableElemSegment seg = js_new(); + if (!seg) { + return false; + } + + ElemSegmentKind kind = flags->kind(); + seg->kind = NormalizeElemSegmentKind(kind); + + if (kind == ElemSegmentKind::Active || + kind == ElemSegmentKind::ActiveWithTableIndex) { + if (env->tables.length() == 0) { + return d.fail("active elem segment requires a table"); + } + + uint32_t tableIndex = 0; + if (kind == ElemSegmentKind::ActiveWithTableIndex && + !d.readVarU32(&tableIndex)) { + return d.fail("expected table index"); + } + if (tableIndex >= env->tables.length()) { + return d.fail("table index out of range for element segment"); + } + seg->tableIndex = tableIndex; + + InitExpr offset; + if (!InitExpr::decodeAndValidate(d, env, ValType::I32, + env->globals.length(), &offset)) { + return false; + } + seg->offsetIfActive.emplace(std::move(offset)); + } else { + // Too many bugs result from keeping this value zero. For passive + // or declared segments, there really is no table index, and we should + // never touch the field. + MOZ_ASSERT(kind == ElemSegmentKind::Passive || + kind == ElemSegmentKind::Declared); + seg->tableIndex = (uint32_t)-1; + } + + ElemSegmentPayload payload = flags->payload(); + RefType elemType; + + // `ActiveWithTableIndex`, `Declared`, and `Passive` element segments encode + // the type or definition kind of the payload. `Active` element segments are + // restricted to MVP behavior, which assumes only function indices. + if (kind == ElemSegmentKind::Active) { + elemType = RefType::func(); + } else { + switch (payload) { + case ElemSegmentPayload::ElemExpression: { + if (!d.readRefType(*env->types, env->features, &elemType)) { + return false; + } + break; + } + case ElemSegmentPayload::ExternIndex: { + uint8_t form; + if (!d.readFixedU8(&form)) { + return d.fail("expected type or extern kind"); + } + + if (form != uint8_t(DefinitionKind::Function)) { + return d.fail( + "segments with extern indices can only contain function " + "references"); + } + elemType = RefType::func(); + } + } + } + + // Check constraints on the element type. + switch (kind) { + case ElemSegmentKind::Active: + case ElemSegmentKind::ActiveWithTableIndex: { + RefType tblElemType = env->tables[seg->tableIndex].elemType; + if (!CheckIsSubtypeOf(d, *env, d.currentOffset(), + ValType(elemType).fieldType(), + ValType(tblElemType).fieldType())) { + return false; + } + break; + } + case ElemSegmentKind::Declared: + case ElemSegmentKind::Passive: { + // Passive segment element types are checked when used with a + // `table.init` instruction. + break; + } + } + seg->elemType = elemType; + + uint32_t numElems; + if (!d.readVarU32(&numElems)) { + return d.fail("expected segment size"); + } + + if (numElems > MaxElemSegmentLength) { + return d.fail("too many table elements"); + } + + if (!seg->elemFuncIndices.reserve(numElems)) { + return false; + } + + bool isAsmJS = seg->active() && env->tables[seg->tableIndex].isAsmJS; + + // For passive segments we should use InitExpr but we don't really want to + // generalize the ElemSection data structure yet, so instead read the + // required Ref.Func and End here. + + for (uint32_t i = 0; i < numElems; i++) { + bool needIndex = true; + + if (payload == ElemSegmentPayload::ElemExpression) { + OpBytes op; + if (!d.readOp(&op)) { + return d.fail("failed to read initializer operation"); + } + + RefType initType = RefType::extern_(); + switch (op.b0) { + case uint16_t(Op::RefFunc): + initType = RefType::func(); + break; + case uint16_t(Op::RefNull): + if (!d.readHeapType(*env->types, env->features, true, &initType)) { + return false; + } + needIndex = false; + break; + default: + return d.fail("failed to read initializer operation"); + } + if (!CheckIsSubtypeOf(d, *env, d.currentOffset(), + ValType(initType).fieldType(), + ValType(elemType).fieldType())) { + return false; + } + } + + uint32_t funcIndex = NullFuncIndex; + if (needIndex) { + if (!d.readVarU32(&funcIndex)) { + return d.fail("failed to read element function index"); + } + if (funcIndex >= env->numFuncs()) { + return d.fail("table element out of range"); + } + } + + if (payload == ElemSegmentPayload::ElemExpression) { + OpBytes end; + if (!d.readOp(&end) || end.b0 != uint16_t(Op::End)) { + return d.fail("failed to read end of initializer expression"); + } + } + + seg->elemFuncIndices.infallibleAppend(funcIndex); + if (funcIndex != NullFuncIndex && !isAsmJS) { + env->declareFuncExported(funcIndex, /* eager */ false, + /* canRefFunc */ true); + } + } + + env->elemSegments.infallibleAppend(std::move(seg)); + } + + return d.finishSection(*range, "elem"); +} + +static bool DecodeDataCountSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::DataCount, env, &range, "datacount")) { + return false; + } + if (!range) { + return true; + } + + uint32_t dataCount; + if (!d.readVarU32(&dataCount)) { + return d.fail("expected data segment count"); + } + + env->dataCount.emplace(dataCount); + + return d.finishSection(*range, "datacount"); +} + +bool wasm::StartsCodeSection(const uint8_t* begin, const uint8_t* end, + SectionRange* codeSection) { + UniqueChars unused; + Decoder d(begin, end, 0, &unused); + + if (!DecodePreamble(d)) { + return false; + } + + while (!d.done()) { + uint8_t id; + SectionRange range; + if (!d.readSectionHeader(&id, &range)) { + return false; + } + + if (id == uint8_t(SectionId::Code)) { + *codeSection = range; + return true; + } + + if (!d.readBytes(range.size)) { + return false; + } + } + + return false; +} + +bool wasm::DecodeModuleEnvironment(Decoder& d, ModuleEnvironment* env) { + if (!DecodePreamble(d)) { + return false; + } + + if (!DecodeTypeSection(d, env)) { + return false; + } + + if (!DecodeImportSection(d, env)) { + return false; + } + + if (!DecodeFunctionSection(d, env)) { + return false; + } + + if (!DecodeTableSection(d, env)) { + return false; + } + + if (!DecodeMemorySection(d, env)) { + return false; + } + + if (!DecodeTagSection(d, env)) { + return false; + } + + if (!DecodeGlobalSection(d, env)) { + return false; + } + + if (!DecodeExportSection(d, env)) { + return false; + } + + if (!DecodeStartSection(d, env)) { + return false; + } + + if (!DecodeElemSection(d, env)) { + return false; + } + + if (!DecodeDataCountSection(d, env)) { + return false; + } + + if (!d.startSection(SectionId::Code, env, &env->codeSection, "code")) { + return false; + } + + if (env->codeSection && env->codeSection->size > MaxCodeSectionBytes) { + return d.fail("code section too big"); + } + + return true; +} + +static bool DecodeFunctionBody(Decoder& d, const ModuleEnvironment& env, + uint32_t funcIndex) { + uint32_t bodySize; + if (!d.readVarU32(&bodySize)) { + return d.fail("expected number of function body bytes"); + } + + if (bodySize > MaxFunctionBytes) { + return d.fail("function body too big"); + } + + if (d.bytesRemain() < bodySize) { + return d.fail("function body length too big"); + } + + return ValidateFunctionBody(env, funcIndex, bodySize, d); +} + +static bool DecodeCodeSection(Decoder& d, ModuleEnvironment* env) { + if (!env->codeSection) { + if (env->numFuncDefs() != 0) { + return d.fail("expected code section"); + } + return true; + } + + uint32_t numFuncDefs; + if (!d.readVarU32(&numFuncDefs)) { + return d.fail("expected function body count"); + } + + if (numFuncDefs != env->numFuncDefs()) { + return d.fail( + "function body count does not match function signature count"); + } + + for (uint32_t funcDefIndex = 0; funcDefIndex < numFuncDefs; funcDefIndex++) { + if (!DecodeFunctionBody(d, *env, env->numFuncImports + funcDefIndex)) { + return false; + } + } + + return d.finishSection(*env->codeSection, "code"); +} + +static bool DecodeDataSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startSection(SectionId::Data, env, &range, "data")) { + return false; + } + if (!range) { + if (env->dataCount.isSome() && *env->dataCount > 0) { + return d.fail("number of data segments does not match declared count"); + } + return true; + } + + uint32_t numSegments; + if (!d.readVarU32(&numSegments)) { + return d.fail("failed to read number of data segments"); + } + + if (numSegments > MaxDataSegments) { + return d.fail("too many data segments"); + } + + if (env->dataCount.isSome() && numSegments != *env->dataCount) { + return d.fail("number of data segments does not match declared count"); + } + + for (uint32_t i = 0; i < numSegments; i++) { + uint32_t initializerKindVal; + if (!d.readVarU32(&initializerKindVal)) { + return d.fail("expected data initializer-kind field"); + } + + switch (initializerKindVal) { + case uint32_t(DataSegmentKind::Active): + case uint32_t(DataSegmentKind::Passive): + case uint32_t(DataSegmentKind::ActiveWithMemoryIndex): + break; + default: + return d.fail("invalid data initializer-kind field"); + } + + DataSegmentKind initializerKind = DataSegmentKind(initializerKindVal); + + if (initializerKind != DataSegmentKind::Passive && !env->usesMemory()) { + return d.fail("active data segment requires a memory section"); + } + + uint32_t memIndex = 0; + if (initializerKind == DataSegmentKind::ActiveWithMemoryIndex) { + if (!d.readVarU32(&memIndex)) { + return d.fail("expected memory index"); + } + if (memIndex > 0) { + return d.fail("memory index must be zero"); + } + } + + DataSegmentEnv seg; + if (initializerKind == DataSegmentKind::Active || + initializerKind == DataSegmentKind::ActiveWithMemoryIndex) { + InitExpr segOffset; + ValType exprType = ToValType(env->memory->indexType()); + if (!InitExpr::decodeAndValidate(d, env, exprType, env->globals.length(), + &segOffset)) { + return false; + } + seg.offsetIfActive.emplace(std::move(segOffset)); + } + + if (!d.readVarU32(&seg.length)) { + return d.fail("expected segment size"); + } + + if (seg.length > MaxDataSegmentLengthPages * PageSize) { + return d.fail("segment size too big"); + } + + seg.bytecodeOffset = d.currentOffset(); + + if (!d.readBytes(seg.length)) { + return d.fail("data segment shorter than declared"); + } + + if (!env->dataSegments.append(std::move(seg))) { + return false; + } + } + + return d.finishSection(*range, "data"); +} + +static bool DecodeModuleNameSubsection(Decoder& d, + const CustomSectionEnv& nameSection, + ModuleEnvironment* env) { + Maybe endOffset; + if (!d.startNameSubsection(NameType::Module, &endOffset)) { + return false; + } + if (!endOffset) { + return true; + } + + Name moduleName; + if (!d.readVarU32(&moduleName.length)) { + return d.fail("failed to read module name length"); + } + + MOZ_ASSERT(d.currentOffset() >= nameSection.payloadOffset); + moduleName.offsetInNamePayload = + d.currentOffset() - nameSection.payloadOffset; + + const uint8_t* bytes; + if (!d.readBytes(moduleName.length, &bytes)) { + return d.fail("failed to read module name bytes"); + } + + if (!d.finishNameSubsection(*endOffset)) { + return false; + } + + // Only save the module name if the whole subsection validates. + env->moduleName.emplace(moduleName); + return true; +} + +static bool DecodeFunctionNameSubsection(Decoder& d, + const CustomSectionEnv& nameSection, + ModuleEnvironment* env) { + Maybe endOffset; + if (!d.startNameSubsection(NameType::Function, &endOffset)) { + return false; + } + if (!endOffset) { + return true; + } + + uint32_t nameCount = 0; + if (!d.readVarU32(&nameCount) || nameCount > MaxFuncs) { + return d.fail("bad function name count"); + } + + NameVector funcNames; + + for (uint32_t i = 0; i < nameCount; ++i) { + uint32_t funcIndex = 0; + if (!d.readVarU32(&funcIndex)) { + return d.fail("unable to read function index"); + } + + // Names must refer to real functions and be given in ascending order. + if (funcIndex >= env->numFuncs() || funcIndex < funcNames.length()) { + return d.fail("invalid function index"); + } + + Name funcName; + if (!d.readVarU32(&funcName.length) || + funcName.length > JS::MaxStringLength) { + return d.fail("unable to read function name length"); + } + + if (!funcName.length) { + continue; + } + + if (!funcNames.resize(funcIndex + 1)) { + return false; + } + + MOZ_ASSERT(d.currentOffset() >= nameSection.payloadOffset); + funcName.offsetInNamePayload = + d.currentOffset() - nameSection.payloadOffset; + + if (!d.readBytes(funcName.length)) { + return d.fail("unable to read function name bytes"); + } + + funcNames[funcIndex] = funcName; + } + + if (!d.finishNameSubsection(*endOffset)) { + return false; + } + + // To encourage fully valid function names subsections; only save names if + // the entire subsection decoded correctly. + env->funcNames = std::move(funcNames); + return true; +} + +static bool DecodeNameSection(Decoder& d, ModuleEnvironment* env) { + MaybeSectionRange range; + if (!d.startCustomSection(NameSectionName, env, &range)) { + return false; + } + if (!range) { + return true; + } + + env->nameCustomSectionIndex = Some(env->customSections.length() - 1); + const CustomSectionEnv& nameSection = env->customSections.back(); + + // Once started, custom sections do not report validation errors. + + if (!DecodeModuleNameSubsection(d, nameSection, env)) { + goto finish; + } + + if (!DecodeFunctionNameSubsection(d, nameSection, env)) { + goto finish; + } + + while (d.currentOffset() < range->end()) { + if (!d.skipNameSubsection()) { + goto finish; + } + } + +finish: + d.finishCustomSection(NameSectionName, *range); + return true; +} + +bool wasm::DecodeModuleTail(Decoder& d, ModuleEnvironment* env) { + if (!DecodeDataSection(d, env)) { + return false; + } + + if (!DecodeNameSection(d, env)) { + return false; + } + + while (!d.done()) { + if (!d.skipCustomSection(env)) { + if (d.resilientMode()) { + d.clearError(); + return true; + } + return false; + } + } + + return true; +} + +// Validate algorithm. + +bool wasm::Validate(JSContext* cx, const ShareableBytes& bytecode, + const FeatureOptions& options, UniqueChars* error) { + Decoder d(bytecode.bytes, 0, error); + + FeatureArgs features = FeatureArgs::build(cx, options); + ModuleEnvironment env(features); + if (!env.init()) { + return false; + } + + if (!DecodeModuleEnvironment(d, &env)) { + return false; + } + + if (!DecodeCodeSection(d, &env)) { + return false; + } + + if (!DecodeModuleTail(d, &env)) { + return false; + } + + MOZ_ASSERT(!*error, "unreported error in decoding"); + return true; +} diff --git a/js/src/wasm/WasmValidate.h b/js/src/wasm/WasmValidate.h new file mode 100644 index 0000000000..8bb2720fad --- /dev/null +++ b/js/src/wasm/WasmValidate.h @@ -0,0 +1,308 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2016 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_validate_h +#define wasm_validate_h + +#include + +#include "js/Utility.h" +#include "js/WasmFeatures.h" + +#include "wasm/WasmBinary.h" +#include "wasm/WasmCompile.h" +#include "wasm/WasmCompileArgs.h" +#include "wasm/WasmModuleTypes.h" +#include "wasm/WasmProcess.h" +#include "wasm/WasmTypeDef.h" + +namespace js { +namespace wasm { + +using mozilla::Some; + +// ModuleEnvironment contains all the state necessary to process or render +// functions, and all of the state necessary to validate all aspects of the +// functions. +// +// A ModuleEnvironment is created by decoding all the sections before the wasm +// code section and then used immutably during. When compiling a module using a +// ModuleGenerator, the ModuleEnvironment holds state shared between the +// ModuleGenerator thread and background compile threads. All the threads +// are given a read-only view of the ModuleEnvironment, thus preventing race +// conditions. + +struct ModuleEnvironment { + // Constant parameters for the entire compilation: + const ModuleKind kind; + const FeatureArgs features; + + // Module fields decoded from the module environment (or initialized while + // validating an asm.js module) and immutable during compilation: + Maybe dataCount; + Maybe memory; + MutableTypeContext types; + FuncDescVector funcs; + uint32_t numFuncImports; + GlobalDescVector globals; + TagDescVector tags; + TableDescVector tables; + Uint32Vector asmJSSigToTableIndex; + ImportVector imports; + ExportVector exports; + Maybe startFuncIndex; + ElemSegmentVector elemSegments; + MaybeSectionRange codeSection; + + // The start offset of the FuncImportInstanceData[] section of the instance + // data. There is one entry for every imported function. + uint32_t funcImportsOffsetStart; + // The start offset of the TypeDefInstanceData[] section of the instance + // data. There is one entry for every type. + uint32_t typeDefsOffsetStart; + // The start offset of the TableInstanceData[] section of the instance data. + // There is one entry for every table. + uint32_t tablesOffsetStart; + // The start offset of the tag section of the instance data. There is one + // entry for every tag. + uint32_t tagsOffsetStart; + + // Fields decoded as part of the wasm module tail: + DataSegmentEnvVector dataSegments; + CustomSectionEnvVector customSections; + Maybe nameCustomSectionIndex; + Maybe moduleName; + NameVector funcNames; + + explicit ModuleEnvironment(FeatureArgs features, + ModuleKind kind = ModuleKind::Wasm) + : kind(kind), + features(features), + memory(Nothing()), + numFuncImports(0), + funcImportsOffsetStart(UINT32_MAX), + typeDefsOffsetStart(UINT32_MAX), + tablesOffsetStart(UINT32_MAX), + tagsOffsetStart(UINT32_MAX) {} + + [[nodiscard]] bool init() { + types = js_new(features); + return types; + } + + size_t numTables() const { return tables.length(); } + size_t numTypes() const { return types->length(); } + size_t numFuncs() const { return funcs.length(); } + size_t numFuncDefs() const { return funcs.length() - numFuncImports; } + + bool funcIsImport(uint32_t funcIndex) const { + return funcIndex < numFuncImports; + } + +#define WASM_FEATURE(NAME, SHORT_NAME, ...) \ + bool SHORT_NAME##Enabled() const { return features.SHORT_NAME; } + JS_FOR_WASM_FEATURES(WASM_FEATURE, WASM_FEATURE, WASM_FEATURE) +#undef WASM_FEATURE + Shareable sharedMemoryEnabled() const { return features.sharedMemory; } + bool hugeMemoryEnabled() const { + return !isAsmJS() && usesMemory() && + IsHugeMemoryEnabled(memory->indexType()); + } + bool simdAvailable() const { return features.simd; } + bool intrinsicsEnabled() const { return features.intrinsics; } + + bool isAsmJS() const { return kind == ModuleKind::AsmJS; } + + bool usesMemory() const { return memory.isSome(); } + bool usesSharedMemory() const { + return memory.isSome() && memory->isShared(); + } + + void declareFuncExported(uint32_t funcIndex, bool eager, bool canRefFunc) { + FuncFlags flags = funcs[funcIndex].flags; + + // Set the `Exported` flag, if not set. + flags = FuncFlags(uint8_t(flags) | uint8_t(FuncFlags::Exported)); + + // Merge in the `Eager` and `CanRefFunc` flags, if they're set. Be sure + // to not unset them if they've already been set. + if (eager) { + flags = FuncFlags(uint8_t(flags) | uint8_t(FuncFlags::Eager)); + } + if (canRefFunc) { + flags = FuncFlags(uint8_t(flags) | uint8_t(FuncFlags::CanRefFunc)); + } + + funcs[funcIndex].flags = flags; + } + + uint32_t offsetOfFuncImportInstanceData(uint32_t funcIndex) const { + MOZ_ASSERT(funcIndex < numFuncImports); + return funcImportsOffsetStart + funcIndex * sizeof(FuncImportInstanceData); + } + + uint32_t offsetOfTypeDefInstanceData(uint32_t typeIndex) const { + MOZ_ASSERT(typeIndex < types->length()); + return typeDefsOffsetStart + typeIndex * sizeof(TypeDefInstanceData); + } + + uint32_t offsetOfTypeDef(uint32_t typeIndex) const { + return offsetOfTypeDefInstanceData(typeIndex) + + offsetof(TypeDefInstanceData, typeDef); + } + uint32_t offsetOfSuperTypeVector(uint32_t typeIndex) const { + return offsetOfTypeDefInstanceData(typeIndex) + + offsetof(TypeDefInstanceData, superTypeVector); + } + + uint32_t offsetOfTableInstanceData(uint32_t tableIndex) const { + MOZ_ASSERT(tableIndex < tables.length()); + return tablesOffsetStart + tableIndex * sizeof(TableInstanceData); + } + + uint32_t offsetOfTagInstanceData(uint32_t tagIndex) const { + MOZ_ASSERT(tagIndex < tags.length()); + return tagsOffsetStart + tagIndex * sizeof(TagInstanceData); + } +}; + +// ElemSegmentFlags provides methods for decoding and encoding the flags field +// of an element segment. This is needed as the flags field has a non-trivial +// encoding that is effectively split into independent `kind` and `payload` +// enums. +class ElemSegmentFlags { + enum class Flags : uint32_t { + Passive = 0x1, + WithIndexOrDeclared = 0x2, + ElemExpression = 0x4, + // Below this line are convenient combinations of flags + KindMask = Passive | WithIndexOrDeclared, + PayloadMask = ElemExpression, + AllFlags = Passive | WithIndexOrDeclared | ElemExpression, + }; + uint32_t encoded_; + + explicit ElemSegmentFlags(uint32_t encoded) : encoded_(encoded) {} + + public: + ElemSegmentFlags(ElemSegmentKind kind, ElemSegmentPayload payload) { + encoded_ = uint32_t(kind) | uint32_t(payload); + } + + static Maybe construct(uint32_t encoded) { + if (encoded > uint32_t(Flags::AllFlags)) { + return Nothing(); + } + return Some(ElemSegmentFlags(encoded)); + } + + uint32_t encoded() const { return encoded_; } + + ElemSegmentKind kind() const { + return static_cast(encoded_ & uint32_t(Flags::KindMask)); + } + ElemSegmentPayload payload() const { + return static_cast(encoded_ & + uint32_t(Flags::PayloadMask)); + } +}; + +// OpIter specialized for validation. + +class NothingVector { + Nothing unused_; + + public: + bool resize(size_t length) { return true; } + Nothing& operator[](size_t) { return unused_; } + Nothing& back() { return unused_; } + size_t length() const { return 0; } + bool append(Nothing& nothing) { return true; } +}; + +struct ValidatingPolicy { + using Value = Nothing; + using ValueVector = NothingVector; + using ControlItem = Nothing; +}; + +template +class OpIter; + +using ValidatingOpIter = OpIter; + +// Shared subtyping function across validation. + +[[nodiscard]] bool CheckIsSubtypeOf(Decoder& d, const ModuleEnvironment& env, + size_t opcodeOffset, FieldType subType, + FieldType superType); + +// The local entries are part of function bodies and thus serialized by both +// wasm and asm.js and decoded as part of both validation and compilation. + +[[nodiscard]] bool EncodeLocalEntries(Encoder& e, const ValTypeVector& locals); + +// This performs no validation; the local entries must already have been +// validated by an earlier pass. + +[[nodiscard]] bool DecodeValidatedLocalEntries(const TypeContext& types, + Decoder& d, + ValTypeVector* locals); + +// This validates the entries. + +[[nodiscard]] bool DecodeLocalEntries(Decoder& d, const TypeContext& types, + const FeatureArgs& features, + ValTypeVector* locals); + +// Returns whether the given [begin, end) prefix of a module's bytecode starts a +// code section and, if so, returns the SectionRange of that code section. +// Note that, even if this function returns 'false', [begin, end) may actually +// be a valid module in the special case when there are no function defs and the +// code section is not present. Such modules can be valid so the caller must +// handle this special case. + +[[nodiscard]] bool StartsCodeSection(const uint8_t* begin, const uint8_t* end, + SectionRange* codeSection); + +// Calling DecodeModuleEnvironment decodes all sections up to the code section +// and performs full validation of all those sections. The client must then +// decode the code section itself, reusing ValidateFunctionBody if necessary, +// and finally call DecodeModuleTail to decode all remaining sections after the +// code section (again, performing full validation). + +[[nodiscard]] bool DecodeModuleEnvironment(Decoder& d, ModuleEnvironment* env); + +[[nodiscard]] bool ValidateFunctionBody(const ModuleEnvironment& env, + uint32_t funcIndex, uint32_t bodySize, + Decoder& d); + +[[nodiscard]] bool DecodeModuleTail(Decoder& d, ModuleEnvironment* env); + +// Validate an entire module, returning true if the module was validated +// successfully. If Validate returns false: +// - if *error is null, the caller should report out-of-memory +// - otherwise, there was a legitimate error described by *error + +[[nodiscard]] bool Validate(JSContext* cx, const ShareableBytes& bytecode, + const FeatureOptions& options, UniqueChars* error); + +} // namespace wasm +} // namespace js + +#endif // namespace wasm_validate_h diff --git a/js/src/wasm/WasmValue.cpp b/js/src/wasm/WasmValue.cpp new file mode 100644 index 0000000000..f05070d1b9 --- /dev/null +++ b/js/src/wasm/WasmValue.cpp @@ -0,0 +1,927 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "wasm/WasmValue.h" + +#include "jsmath.h" +#include "js/friend/ErrorMessages.h" // JSMSG_* +#include "js/Printf.h" +#include "js/Value.h" +#include "vm/BigIntType.h" +#include "vm/GlobalObject.h" +#include "vm/JSContext.h" +#include "vm/JSObject.h" +#include "vm/StringType.h" +#include "wasm/WasmGcObject.h" +#include "wasm/WasmJS.h" +#include "wasm/WasmLog.h" + +#include "vm/JSObject-inl.h" + +using namespace js; +using namespace js::wasm; + +Val::Val(const LitVal& val) { + type_ = val.type(); + switch (type_.kind()) { + case ValType::I32: + cell_.i32_ = val.i32(); + return; + case ValType::F32: + cell_.f32_ = val.f32(); + return; + case ValType::I64: + cell_.i64_ = val.i64(); + return; + case ValType::F64: + cell_.f64_ = val.f64(); + return; + case ValType::V128: + cell_.v128_ = val.v128(); + return; + case ValType::Ref: + cell_.ref_ = val.ref(); + return; + } + MOZ_CRASH(); +} + +void Val::readFromRootedLocation(const void* loc) { + memset(&cell_, 0, sizeof(Cell)); + memcpy(&cell_, loc, type_.size()); +} + +void Val::initFromRootedLocation(ValType type, const void* loc) { + MOZ_ASSERT(!type_.isValid()); + type_ = type; + memset(&cell_, 0, sizeof(Cell)); + memcpy(&cell_, loc, type_.size()); +} + +void Val::initFromHeapLocation(ValType type, const void* loc) { + MOZ_ASSERT(!type_.isValid()); + type_ = type; + memset(&cell_, 0, sizeof(Cell)); + readFromHeapLocation(loc); +} + +void Val::writeToRootedLocation(void* loc, bool mustWrite64) const { + memcpy(loc, &cell_, type_.size()); + if (mustWrite64 && type_.size() == 4) { + memset((uint8_t*)(loc) + 4, 0, 4); + } +} + +void Val::readFromHeapLocation(const void* loc) { + memcpy(&cell_, loc, type_.size()); +} + +void Val::writeToHeapLocation(void* loc) const { + if (type_.isRefRepr()) { + // TODO/AnyRef-boxing: With boxed immediates and strings, the write + // barrier is going to have to be more complicated. + *((GCPtr*)loc) = cell_.ref_.asJSObject(); + return; + } + memcpy(loc, &cell_, type_.size()); +} + +bool Val::fromJSValue(JSContext* cx, ValType targetType, HandleValue val, + MutableHandleVal rval) { + rval.get().type_ = targetType; + // No pre/post barrier needed as rval is rooted + return ToWebAssemblyValue(cx, val, targetType, &rval.get().cell_, + targetType.size() == 8); +} + +bool Val::toJSValue(JSContext* cx, MutableHandleValue rval) const { + return ToJSValue(cx, &cell_, type_, rval); +} + +void Val::trace(JSTracer* trc) const { + if (isJSObject()) { + // TODO/AnyRef-boxing: With boxed immediates and strings, the write + // barrier is going to have to be more complicated. + ASSERT_ANYREF_IS_JSOBJECT; + TraceManuallyBarrieredEdge(trc, asJSObjectAddress(), "wasm val"); + } +} + +bool wasm::CheckRefType(JSContext* cx, RefType targetType, HandleValue v, + MutableHandleFunction fnval, + MutableHandleAnyRef refval) { + if (!targetType.isNullable() && v.isNull()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_REF_NONNULLABLE_VALUE); + return false; + } + + switch (targetType.kind()) { + case RefType::Func: + return CheckFuncRefValue(cx, v, fnval); + case RefType::Extern: + return BoxAnyRef(cx, v, refval); + case RefType::Any: + return CheckAnyRefValue(cx, v, refval); + case RefType::NoFunc: + return CheckNullFuncRefValue(cx, v, fnval); + case RefType::NoExtern: + return CheckNullExternRefValue(cx, v, refval); + case RefType::None: + return CheckNullRefValue(cx, v, refval); + case RefType::Eq: + return CheckEqRefValue(cx, v, refval); + case RefType::Struct: + return CheckStructRefValue(cx, v, refval); + case RefType::Array: + return CheckArrayRefValue(cx, v, refval); + case RefType::TypeRef: + return CheckTypeRefValue(cx, targetType.typeDef(), v, refval); + } + + MOZ_ASSERT(!ValType(targetType).isExposable()); + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; +} + +bool wasm::CheckFuncRefValue(JSContext* cx, HandleValue v, + MutableHandleFunction fun) { + if (v.isNull()) { + MOZ_ASSERT(!fun); + return true; + } + + if (v.isObject()) { + JSObject& obj = v.toObject(); + if (obj.is()) { + JSFunction* f = &obj.as(); + if (IsWasmExportedFunction(f)) { + fun.set(f); + return true; + } + } + } + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_FUNCREF_VALUE); + return false; +} + +bool wasm::CheckAnyRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp) { + if (v.isNull()) { + vp.set(AnyRef::null()); + return true; + } + + if (v.isObject()) { + JSObject& obj = v.toObject(); + if (obj.is()) { + vp.set(AnyRef::fromJSObject(&obj.as())); + return true; + } + } + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_ANYREF_VALUE); + return false; +} + +bool wasm::CheckNullFuncRefValue(JSContext* cx, HandleValue v, + MutableHandleFunction fun) { + if (!v.isNull()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_NULL_FUNCREF_VALUE); + return false; + } + MOZ_ASSERT(!fun); + return true; +} + +bool wasm::CheckNullExternRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp) { + if (!v.isNull()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_NULL_EXTERNREF_VALUE); + return false; + } + + vp.set(AnyRef::null()); + return true; +} + +bool wasm::CheckNullRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp) { + if (!v.isNull()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_NULL_ANYREF_VALUE); + return false; + } + + vp.set(AnyRef::null()); + return true; +} + +bool wasm::CheckEqRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp) { + if (v.isNull()) { + vp.set(AnyRef::null()); + return true; + } + + if (v.isObject()) { + JSObject& obj = v.toObject(); + if (obj.is()) { + vp.set(AnyRef::fromJSObject(&obj.as())); + return true; + } + } + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_EQREF_VALUE); + return false; +} + +bool wasm::CheckStructRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp) { + if (v.isNull()) { + vp.set(AnyRef::null()); + return true; + } + + if (v.isObject()) { + JSObject& obj = v.toObject(); + if (obj.is()) { + vp.set(AnyRef::fromJSObject(&obj.as())); + return true; + } + } + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_STRUCTREF_VALUE); + return false; +} + +bool wasm::CheckArrayRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp) { + if (v.isNull()) { + vp.set(AnyRef::null()); + return true; + } + + if (v.isObject()) { + JSObject& obj = v.toObject(); + if (obj.is()) { + vp.set(AnyRef::fromJSObject(&obj.as())); + return true; + } + } + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_ARRAYREF_VALUE); + return false; +} + +bool wasm::CheckTypeRefValue(JSContext* cx, const TypeDef* typeDef, + HandleValue v, MutableHandleAnyRef vp) { + if (v.isNull()) { + vp.set(AnyRef::null()); + return true; + } + + if (v.isObject()) { + JSObject& obj = v.toObject(); + if (obj.is() && + obj.as().isRuntimeSubtypeOf(typeDef)) { + vp.set(AnyRef::fromJSObject(&obj.as())); + return true; + } + } + + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_TYPEREF_VALUE); + return false; +} + +class wasm::NoDebug { + public: + template + static void print(T v) {} +}; + +class wasm::DebugCodegenVal { + template + static void print(const char* fmt, T v) { + DebugCodegen(DebugChannel::Function, fmt, v); + } + + public: + static void print(int32_t v) { print(" i32(%d)", v); } + static void print(int64_t v) { print(" i64(%" PRId64 ")", v); } + static void print(float v) { print(" f32(%f)", v); } + static void print(double v) { print(" f64(%lf)", v); } + static void print(void* v) { print(" ptr(%p)", v); } +}; + +template bool wasm::ToWebAssemblyValue(JSContext* cx, HandleValue val, + FieldType type, void* loc, + bool mustWrite64, + CoercionLevel level); +template bool wasm::ToWebAssemblyValue( + JSContext* cx, HandleValue val, FieldType type, void* loc, bool mustWrite64, + CoercionLevel level); +template bool wasm::ToJSValue(JSContext* cx, const void* src, + FieldType type, MutableHandleValue dst, + CoercionLevel level); +template bool wasm::ToJSValue(JSContext* cx, const void* src, + FieldType type, + MutableHandleValue dst, + CoercionLevel level); +template bool wasm::ToJSValueMayGC(FieldType type); +template bool wasm::ToJSValueMayGC(FieldType type); + +template bool wasm::ToWebAssemblyValue(JSContext* cx, HandleValue val, + ValType type, void* loc, + bool mustWrite64, + CoercionLevel level); +template bool wasm::ToWebAssemblyValue(JSContext* cx, + HandleValue val, + ValType type, void* loc, + bool mustWrite64, + CoercionLevel level); +template bool wasm::ToJSValue(JSContext* cx, const void* src, + ValType type, MutableHandleValue dst, + CoercionLevel level); +template bool wasm::ToJSValue(JSContext* cx, const void* src, + ValType type, + MutableHandleValue dst, + CoercionLevel level); +template bool wasm::ToJSValueMayGC(ValType type); +template bool wasm::ToJSValueMayGC(ValType type); + +template +bool ToWebAssemblyValue_i8(JSContext* cx, HandleValue val, int8_t* loc) { + bool ok = ToInt8(cx, val, loc); + Debug::print(*loc); + return ok; +} + +template +bool ToWebAssemblyValue_i16(JSContext* cx, HandleValue val, int16_t* loc) { + bool ok = ToInt16(cx, val, loc); + Debug::print(*loc); + return ok; +} + +template +bool ToWebAssemblyValue_i32(JSContext* cx, HandleValue val, int32_t* loc, + bool mustWrite64) { + bool ok = ToInt32(cx, val, loc); + if (ok && mustWrite64) { +#if defined(JS_CODEGEN_MIPS64) + loc[1] = loc[0] >> 31; +#else + loc[1] = 0; +#endif + } + Debug::print(*loc); + return ok; +} + +template +bool ToWebAssemblyValue_i64(JSContext* cx, HandleValue val, int64_t* loc, + bool mustWrite64) { + MOZ_ASSERT(mustWrite64); + JS_TRY_VAR_OR_RETURN_FALSE(cx, *loc, ToBigInt64(cx, val)); + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_f32(JSContext* cx, HandleValue val, float* loc, + bool mustWrite64) { + bool ok = RoundFloat32(cx, val, loc); + if (ok && mustWrite64) { + loc[1] = 0.0; + } + Debug::print(*loc); + return ok; +} + +template +bool ToWebAssemblyValue_f64(JSContext* cx, HandleValue val, double* loc, + bool mustWrite64) { + MOZ_ASSERT(mustWrite64); + bool ok = ToNumber(cx, val, loc); + Debug::print(*loc); + return ok; +} + +template +bool ToWebAssemblyValue_externref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!BoxAnyRef(cx, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_nullexternref(JSContext* cx, HandleValue val, + void** loc, bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!CheckNullExternRefValue(cx, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_funcref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedFunction fun(cx); + if (!CheckFuncRefValue(cx, val, &fun)) { + return false; + } + loc[0] = fun; +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_nullfuncref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedFunction fun(cx); + if (!CheckNullFuncRefValue(cx, val, &fun)) { + return false; + } + loc[0] = fun; +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_anyref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!CheckAnyRefValue(cx, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_nullref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!CheckNullRefValue(cx, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_eqref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + return ToWebAssemblyValue_anyref(cx, val, loc, mustWrite64); +} + +template +bool ToWebAssemblyValue_structref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!CheckStructRefValue(cx, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_arrayref(JSContext* cx, HandleValue val, void** loc, + bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!CheckArrayRefValue(cx, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +template +bool ToWebAssemblyValue_typeref(JSContext* cx, const TypeDef* typeDef, + HandleValue val, void** loc, bool mustWrite64) { + RootedAnyRef result(cx, AnyRef::null()); + if (!CheckTypeRefValue(cx, typeDef, val, &result)) { + return false; + } + loc[0] = result.get().forCompiledCode(); +#ifndef JS_64BIT + if (mustWrite64) { + loc[1] = nullptr; + } +#endif + Debug::print(*loc); + return true; +} + +bool ToWebAssemblyValue_lossless(JSContext* cx, HandleValue val, ValType type, + void* loc, bool mustWrite64) { + if (!val.isObject() || !val.toObject().is()) { + return false; + } + Rooted srcVal(cx, &val.toObject().as()); + + if (srcVal->type() != type) { + return false; + } + + srcVal->val().get().writeToRootedLocation(loc, mustWrite64); + return true; +} + +template +bool wasm::ToWebAssemblyValue(JSContext* cx, HandleValue val, FieldType type, + void* loc, bool mustWrite64, + CoercionLevel level) { + if (level == CoercionLevel::Lossless && + ToWebAssemblyValue_lossless(cx, val, type.valType(), (void*)loc, + mustWrite64)) { + return true; + } + + switch (type.kind()) { + case FieldType::I8: + return ToWebAssemblyValue_i8(cx, val, (int8_t*)loc); + case FieldType::I16: + return ToWebAssemblyValue_i16(cx, val, (int16_t*)loc); + case FieldType::I32: + return ToWebAssemblyValue_i32(cx, val, (int32_t*)loc, mustWrite64); + case FieldType::I64: + return ToWebAssemblyValue_i64(cx, val, (int64_t*)loc, mustWrite64); + case FieldType::F32: + return ToWebAssemblyValue_f32(cx, val, (float*)loc, mustWrite64); + case FieldType::F64: + return ToWebAssemblyValue_f64(cx, val, (double*)loc, mustWrite64); + case FieldType::V128: + break; + case FieldType::Ref: +#ifdef ENABLE_WASM_FUNCTION_REFERENCES + if (!type.isNullable() && val.isNull()) { + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_REF_NONNULLABLE_VALUE); + return false; + } +#else + MOZ_ASSERT(type.isNullable()); +#endif + switch (type.refTypeKind()) { + case RefType::Func: + return ToWebAssemblyValue_funcref(cx, val, (void**)loc, + mustWrite64); + case RefType::Extern: + return ToWebAssemblyValue_externref(cx, val, (void**)loc, + mustWrite64); + case RefType::Any: + return ToWebAssemblyValue_anyref(cx, val, (void**)loc, + mustWrite64); + case RefType::NoFunc: + return ToWebAssemblyValue_nullfuncref(cx, val, (void**)loc, + mustWrite64); + case RefType::NoExtern: + return ToWebAssemblyValue_nullexternref(cx, val, (void**)loc, + mustWrite64); + case RefType::None: + return ToWebAssemblyValue_nullref(cx, val, (void**)loc, + mustWrite64); + case RefType::Eq: + return ToWebAssemblyValue_eqref(cx, val, (void**)loc, + mustWrite64); + case RefType::Struct: + return ToWebAssemblyValue_structref(cx, val, (void**)loc, + mustWrite64); + case RefType::Array: + return ToWebAssemblyValue_arrayref(cx, val, (void**)loc, + mustWrite64); + case RefType::TypeRef: + return ToWebAssemblyValue_typeref(cx, type.typeDef(), val, + (void**)loc, mustWrite64); + } + } + + MOZ_ASSERT(!type.isExposable()); + JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, + JSMSG_WASM_BAD_VAL_TYPE); + return false; +} + +template +bool wasm::ToWebAssemblyValue(JSContext* cx, HandleValue val, ValType type, + void* loc, bool mustWrite64, + CoercionLevel level) { + return wasm::ToWebAssemblyValue(cx, val, FieldType(type.packed()), loc, + mustWrite64, level); +} + +template +bool ToJSValue_i8(JSContext* cx, int8_t src, MutableHandleValue dst) { + dst.set(Int32Value(src)); + Debug::print(src); + return true; +} + +template +bool ToJSValue_i16(JSContext* cx, int16_t src, MutableHandleValue dst) { + dst.set(Int32Value(src)); + Debug::print(src); + return true; +} + +template +bool ToJSValue_i32(JSContext* cx, int32_t src, MutableHandleValue dst) { + dst.set(Int32Value(src)); + Debug::print(src); + return true; +} + +template +bool ToJSValue_i64(JSContext* cx, int64_t src, MutableHandleValue dst) { + // If bi is manipulated other than test & storing, it would need + // to be rooted here. + BigInt* bi = BigInt::createFromInt64(cx, src); + if (!bi) { + return false; + } + dst.set(BigIntValue(bi)); + Debug::print(src); + return true; +} + +template +bool ToJSValue_f32(JSContext* cx, float src, MutableHandleValue dst) { + dst.set(JS::CanonicalizedDoubleValue(src)); + Debug::print(src); + return true; +} + +template +bool ToJSValue_f64(JSContext* cx, double src, MutableHandleValue dst) { + dst.set(JS::CanonicalizedDoubleValue(src)); + Debug::print(src); + return true; +} + +template +bool ToJSValue_funcref(JSContext* cx, void* src, MutableHandleValue dst) { + dst.set(UnboxFuncRef(FuncRef::fromCompiledCode(src))); + Debug::print(src); + return true; +} + +template +bool ToJSValue_externref(JSContext* cx, void* src, MutableHandleValue dst) { + dst.set(UnboxAnyRef(AnyRef::fromCompiledCode(src))); + Debug::print(src); + return true; +} + +template +bool ToJSValue_anyref(JSContext* cx, void* src, MutableHandleValue dst) { + dst.set(UnboxAnyRef(AnyRef::fromCompiledCode(src))); + Debug::print(src); + return true; +} + +template +bool ToJSValue_lossless(JSContext* cx, const void* src, MutableHandleValue dst, + ValType type) { + RootedVal srcVal(cx); + srcVal.get().initFromRootedLocation(type, src); + RootedObject prototype( + cx, GlobalObject::getOrCreatePrototype(cx, JSProto_WasmGlobal)); + Rooted srcGlobal( + cx, WasmGlobalObject::create(cx, srcVal, false, prototype)); + dst.set(ObjectValue(*srcGlobal.get())); + return true; +} + +template +bool wasm::ToJSValue(JSContext* cx, const void* src, FieldType type, + MutableHandleValue dst, CoercionLevel level) { + if (level == CoercionLevel::Lossless) { + MOZ_ASSERT(type.isValType()); + return ToJSValue_lossless(cx, src, dst, type.valType()); + } + + switch (type.kind()) { + case FieldType::I8: + return ToJSValue_i8(cx, *reinterpret_cast(src), + dst); + case FieldType::I16: + return ToJSValue_i16(cx, *reinterpret_cast(src), + dst); + case FieldType::I32: + return ToJSValue_i32(cx, *reinterpret_cast(src), + dst); + case FieldType::I64: + return ToJSValue_i64(cx, *reinterpret_cast(src), + dst); + case FieldType::F32: + return ToJSValue_f32(cx, *reinterpret_cast(src), + dst); + case FieldType::F64: + return ToJSValue_f64(cx, *reinterpret_cast(src), + dst); + case FieldType::V128: + break; + case FieldType::Ref: + switch (type.refType().hierarchy()) { + case RefTypeHierarchy::Func: + return ToJSValue_funcref( + cx, *reinterpret_cast(src), dst); + case RefTypeHierarchy::Extern: + return ToJSValue_externref( + cx, *reinterpret_cast(src), dst); + case RefTypeHierarchy::Any: + return ToJSValue_anyref( + cx, *reinterpret_cast(src), dst); + break; + } + } + MOZ_ASSERT(!type.isExposable()); + Debug::print(nullptr); + dst.setUndefined(); + return true; +} + +template +bool wasm::ToJSValueMayGC(FieldType type) { + return type.kind() == FieldType::I64; +} + +template +bool wasm::ToJSValue(JSContext* cx, const void* src, ValType type, + MutableHandleValue dst, CoercionLevel level) { + return wasm::ToJSValue(cx, src, FieldType(type.packed()), dst, level); +} + +template +bool wasm::ToJSValueMayGC(ValType type) { + return wasm::ToJSValueMayGC(FieldType(type.packed())); +} + +void AnyRef::trace(JSTracer* trc) { + if (value_) { + TraceManuallyBarrieredEdge(trc, &value_, "wasm anyref referent"); + } +} + +const JSClass WasmValueBox::class_ = { + "WasmValueBox", JSCLASS_HAS_RESERVED_SLOTS(RESERVED_SLOTS)}; + +WasmValueBox* WasmValueBox::create(JSContext* cx, HandleValue val) { + WasmValueBox* obj = NewObjectWithGivenProto(cx, nullptr); + if (!obj) { + return nullptr; + } + obj->setFixedSlot(VALUE_SLOT, val); + return obj; +} + +bool wasm::BoxAnyRef(JSContext* cx, HandleValue val, + MutableHandleAnyRef result) { + if (val.isNull()) { + result.set(AnyRef::null()); + return true; + } + + if (val.isObject()) { + JSObject* obj = &val.toObject(); + MOZ_ASSERT(!obj->is()); + MOZ_ASSERT(obj->compartment() == cx->compartment()); + result.set(AnyRef::fromJSObject(obj)); + return true; + } + + WasmValueBox* box = WasmValueBox::create(cx, val); + if (!box) return false; + result.set(AnyRef::fromJSObject(box)); + return true; +} + +JSObject* wasm::BoxBoxableValue(JSContext* cx, HandleValue val) { + MOZ_ASSERT(!val.isNull() && !val.isObject()); + return WasmValueBox::create(cx, val); +} + +Value wasm::UnboxAnyRef(AnyRef val) { + // If UnboxAnyRef needs to allocate then we need a more complicated API, and + // we need to root the value in the callers, see comments in callExport(). + JSObject* obj = val.asJSObject(); + Value result; + if (obj == nullptr) { + result.setNull(); + } else if (obj->is()) { + result = obj->as().value(); + } else { + result.setObjectOrNull(obj); + } + return result; +} + +/* static */ +wasm::FuncRef wasm::FuncRef::fromAnyRefUnchecked(AnyRef p) { +#ifdef DEBUG + Value v = UnboxAnyRef(p); + if (v.isNull()) { + return FuncRef(nullptr); + } + if (v.toObject().is()) { + return FuncRef(&v.toObject().as()); + } + MOZ_CRASH("Bad value"); +#else + return FuncRef(&p.asJSObject()->as()); +#endif +} + +void wasm::FuncRef::trace(JSTracer* trc) const { + if (value_) { + TraceManuallyBarrieredEdge(trc, &value_, "wasm funcref referent"); + } +} + +Value wasm::UnboxFuncRef(FuncRef val) { + JSFunction* fn = val.asJSFunction(); + Value result; + MOZ_ASSERT_IF(fn, fn->is()); + result.setObjectOrNull(fn); + return result; +} diff --git a/js/src/wasm/WasmValue.h b/js/src/wasm/WasmValue.h new file mode 100644 index 0000000000..26ff65e327 --- /dev/null +++ b/js/src/wasm/WasmValue.h @@ -0,0 +1,652 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * + * Copyright 2021 Mozilla Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef wasm_val_h +#define wasm_val_h + +#include "js/Class.h" // JSClassOps, ClassSpec +#include "vm/JSObject.h" +#include "vm/NativeObject.h" // NativeObject +#include "wasm/WasmSerialize.h" +#include "wasm/WasmTypeDef.h" + +namespace js { +namespace wasm { + +// A V128 value. + +struct V128 { + uint8_t bytes[16] = {}; // Little-endian + + WASM_CHECK_CACHEABLE_POD(bytes); + + V128() = default; + + explicit V128(uint8_t splatValue) { + memset(bytes, int(splatValue), sizeof(bytes)); + } + + template + void extractLane(unsigned lane, T* result) const { + MOZ_ASSERT(lane < 16 / sizeof(T)); + memcpy(result, bytes + sizeof(T) * lane, sizeof(T)); + } + + template + void insertLane(unsigned lane, T value) { + MOZ_ASSERT(lane < 16 / sizeof(T)); + memcpy(bytes + sizeof(T) * lane, &value, sizeof(T)); + } + + bool operator==(const V128& rhs) const { + for (size_t i = 0; i < sizeof(bytes); i++) { + if (bytes[i] != rhs.bytes[i]) { + return false; + } + } + return true; + } + + bool operator!=(const V128& rhs) const { return !(*this == rhs); } +}; + +WASM_DECLARE_CACHEABLE_POD(V128); + +static_assert(sizeof(V128) == 16, "Invariant"); + +// An AnyRef is a boxed value that can represent any wasm reference type and any +// host type that the host system allows to flow into and out of wasm +// transparently. It is a pointer-sized datum that has the same representation +// as all its subtypes (funcref, externref, eqref, (ref T), et al) due to the +// non-coercive subtyping of the wasm type system. Its current representation +// is a plain JSObject*, and the private JSObject subtype WasmValueBox is used +// to box non-object non-null JS values. +// +// The C++/wasm boundary always uses a 'void*' type to express AnyRef values, to +// emphasize the pointer-ness of the value. The C++ code must transform the +// void* into an AnyRef by calling AnyRef::fromCompiledCode(), and transform an +// AnyRef into a void* by calling AnyRef::toCompiledCode(). Once in C++, we use +// AnyRef everywhere. A JS Value is transformed into an AnyRef by calling +// AnyRef::box(), and the AnyRef is transformed into a JS Value by calling +// AnyRef::unbox(). +// +// NOTE that AnyRef values may point to GC'd storage and as such need to be +// rooted if they are kept live in boxed form across code that may cause GC! +// Use RootedAnyRef / HandleAnyRef / MutableHandleAnyRef where necessary. +// +// The lowest bits of the pointer value are used for tagging, to allow for some +// representation optimizations and to distinguish various types. + +// For version 0, we simply equate AnyRef and JSObject* (this means that there +// are technically no tags at all yet). We use a simple boxing scheme that +// wraps a JS value that is not already JSObject in a distinguishable JSObject +// that holds the value, see WasmTypes.cpp for details. Knowledge of this +// mapping is embedded in CodeGenerator.cpp (in WasmBoxValue and +// WasmAnyRefFromJSObject) and in WasmStubs.cpp (in functions Box* and Unbox*). + +class AnyRef { + // mutable so that tracing may access a JSObject* from a `const Val` or + // `const AnyRef`. + mutable JSObject* value_; + + explicit AnyRef() : value_((JSObject*)-1) {} + explicit AnyRef(JSObject* p) : value_(p) { + MOZ_ASSERT(((uintptr_t)p & 0x03) == 0); + } + + public: + // An invalid AnyRef cannot arise naturally from wasm and so can be used as + // a sentinel value to indicate failure from an AnyRef-returning function. + static AnyRef invalid() { return AnyRef(); } + + // Given a void* that comes from compiled wasm code, turn it into AnyRef. + static AnyRef fromCompiledCode(void* p) { return AnyRef((JSObject*)p); } + + // Given a JSObject* that comes from JS, turn it into AnyRef. + static AnyRef fromJSObject(JSObject* p) { return AnyRef(p); } + + // Generate an AnyRef null pointer. + static AnyRef null() { return AnyRef(nullptr); } + + bool isNull() const { return value_ == nullptr; } + + bool operator==(const AnyRef& rhs) const { + return this->value_ == rhs.value_; + } + + bool operator!=(const AnyRef& rhs) const { return !(*this == rhs); } + + void* forCompiledCode() const { return value_; } + + JSObject* asJSObject() const { return value_; } + + JSObject** asJSObjectAddress() const { return &value_; } + + void trace(JSTracer* trc); + + // Tags (to be developed further) + static constexpr uintptr_t AnyRefTagMask = 1; + static constexpr uintptr_t AnyRefObjTag = 0; +}; + +using RootedAnyRef = Rooted; +using HandleAnyRef = Handle; +using MutableHandleAnyRef = MutableHandle; + +// TODO/AnyRef-boxing: With boxed immediates and strings, these will be defined +// as MOZ_CRASH or similar so that we can find all locations that need to be +// fixed. + +#define ASSERT_ANYREF_IS_JSOBJECT (void)(0) +#define STATIC_ASSERT_ANYREF_IS_JSOBJECT static_assert(1, "AnyRef is JSObject") + +// Given any JS value, box it as an AnyRef and store it in *result. Returns +// false on OOM. + +bool BoxAnyRef(JSContext* cx, HandleValue val, MutableHandleAnyRef result); + +// Given a JS value that requires an object box, box it as an AnyRef and return +// it, returning nullptr on OOM. +// +// Currently the values requiring a box are those other than JSObject* or +// nullptr, but in the future more values will be represented without an +// allocation. +JSObject* BoxBoxableValue(JSContext* cx, HandleValue val); + +// Given any AnyRef, unbox it as a JS Value. If it is a reference to a wasm +// object it will be reflected as a JSObject* representing some TypedObject +// instance. + +Value UnboxAnyRef(AnyRef val); + +class WasmValueBox : public NativeObject { + static const unsigned VALUE_SLOT = 0; + + public: + static const unsigned RESERVED_SLOTS = 1; + static const JSClass class_; + + static WasmValueBox* create(JSContext* cx, HandleValue val); + Value value() const { return getFixedSlot(VALUE_SLOT); } + static size_t offsetOfValue() { + return NativeObject::getFixedSlotOffset(VALUE_SLOT); + } +}; + +// A FuncRef is a JSFunction* and is hence also an AnyRef, and the remarks above +// about AnyRef apply also to FuncRef. When 'funcref' is used as a value type +// in wasm code, the value that is held is "the canonical function value", which +// is a function for which IsWasmExportedFunction() is true, and which has the +// correct identity wrt reference equality of functions. Notably, if a function +// is imported then its ref.func value compares === in JS to the function that +// was passed as an import when the instance was created. +// +// These rules ensure that casts from funcref to anyref are non-converting +// (generate no code), and that no wrapping or unwrapping needs to happen when a +// funcref or anyref flows across the JS/wasm boundary, and that functions have +// the necessary identity when observed from JS, and in the future, from wasm. +// +// Functions stored in tables, whether wasm tables or internal tables, can be +// stored in a form that optimizes for eg call speed, however. +// +// Reading a funcref from a funcref table, writing a funcref to a funcref table, +// and generating the value for a ref.func instruction are therefore nontrivial +// operations that require mapping between the canonical JSFunction and the +// optimized table representation. Once we get an instruction to call a +// ref.func directly it too will require such a mapping. + +// In many cases, a FuncRef is exactly the same as AnyRef and we can use AnyRef +// functionality on funcref values. The FuncRef class exists mostly to add more +// checks and to make it clear, when we need to, that we're manipulating funcref +// values. FuncRef does not currently subclass AnyRef because there's been no +// need to, but it probably could. + +class FuncRef { + // mutable so that tracing may access a JSFunction* from a `const FuncRef` + mutable JSFunction* value_; + + explicit FuncRef() : value_((JSFunction*)-1) {} + explicit FuncRef(JSFunction* p) : value_(p) { + MOZ_ASSERT(((uintptr_t)p & 0x03) == 0); + } + + public: + // Given a void* that comes from compiled wasm code, turn it into FuncRef. + static FuncRef fromCompiledCode(void* p) { return FuncRef((JSFunction*)p); } + + // Given a JSFunction* that comes from JS, turn it into FuncRef. + static FuncRef fromJSFunction(JSFunction* p) { return FuncRef(p); } + + // Given an AnyRef that represents a possibly-null funcref, turn it into a + // FuncRef. + static FuncRef fromAnyRefUnchecked(AnyRef p); + + AnyRef asAnyRef() { return AnyRef::fromJSObject((JSObject*)value_); } + + void* forCompiledCode() const { return value_; } + + JSFunction* asJSFunction() { return value_; } + + bool isNull() const { return value_ == nullptr; } + + void trace(JSTracer* trc) const; +}; + +using RootedFuncRef = Rooted; +using HandleFuncRef = Handle; +using MutableHandleFuncRef = MutableHandle; + +// Given any FuncRef, unbox it as a JS Value -- always a JSFunction*. + +Value UnboxFuncRef(FuncRef val); + +// The LitVal class represents a single WebAssembly value of a given value +// type, mostly for the purpose of numeric literals and initializers. A LitVal +// does not directly map to a JS value since there is not (currently) a precise +// representation of i64 values. A LitVal may contain non-canonical NaNs since, +// within WebAssembly, floats are not canonicalized. Canonicalization must +// happen at the JS boundary. + +class LitVal { + public: + union Cell { + uint32_t i32_; + uint64_t i64_; + float f32_; + double f64_; + wasm::V128 v128_; + wasm::AnyRef ref_; + + Cell() : v128_() {} + ~Cell() = default; + + WASM_CHECK_CACHEABLE_POD(i32_, i64_, f32_, f64_, v128_); + WASM_ALLOW_NON_CACHEABLE_POD_FIELD( + ref_, + "The pointer value in ref_ is guaranteed to always be null in a " + "LitVal."); + }; + + protected: + ValType type_; + Cell cell_; + + public: + LitVal() : type_(ValType()), cell_{} {} + + explicit LitVal(ValType type) : type_(type) { + switch (type.kind()) { + case ValType::Kind::I32: { + cell_.i32_ = 0; + break; + } + case ValType::Kind::I64: { + cell_.i64_ = 0; + break; + } + case ValType::Kind::F32: { + cell_.f32_ = 0; + break; + } + case ValType::Kind::F64: { + cell_.f64_ = 0; + break; + } + case ValType::Kind::V128: { + new (&cell_.v128_) V128(); + break; + } + case ValType::Kind::Ref: { + cell_.ref_ = AnyRef::null(); + break; + } + } + } + + explicit LitVal(uint32_t i32) : type_(ValType::I32) { cell_.i32_ = i32; } + explicit LitVal(uint64_t i64) : type_(ValType::I64) { cell_.i64_ = i64; } + + explicit LitVal(float f32) : type_(ValType::F32) { cell_.f32_ = f32; } + explicit LitVal(double f64) : type_(ValType::F64) { cell_.f64_ = f64; } + + explicit LitVal(V128 v128) : type_(ValType::V128) { cell_.v128_ = v128; } + + explicit LitVal(ValType type, AnyRef any) : type_(type) { + MOZ_ASSERT(type.isRefRepr()); + MOZ_ASSERT(any.isNull(), + "use Val for non-nullptr ref types to get tracing"); + cell_.ref_ = any; + } + + ValType type() const { return type_; } + static constexpr size_t sizeofLargestValue() { return sizeof(cell_); } + + Cell& cell() { return cell_; } + const Cell& cell() const { return cell_; } + + uint32_t i32() const { + MOZ_ASSERT(type_ == ValType::I32); + return cell_.i32_; + } + uint64_t i64() const { + MOZ_ASSERT(type_ == ValType::I64); + return cell_.i64_; + } + const float& f32() const { + MOZ_ASSERT(type_ == ValType::F32); + return cell_.f32_; + } + const double& f64() const { + MOZ_ASSERT(type_ == ValType::F64); + return cell_.f64_; + } + AnyRef ref() const { + MOZ_ASSERT(type_.isRefRepr()); + return cell_.ref_; + } + const V128& v128() const { + MOZ_ASSERT(type_ == ValType::V128); + return cell_.v128_; + } + + WASM_DECLARE_FRIEND_SERIALIZE(LitVal); +}; + +WASM_DECLARE_CACHEABLE_POD(LitVal::Cell); + +// A Val is a LitVal that can contain (non-null) pointers to GC things. All Vals +// must be used with the rooting APIs as they may contain JS objects. + +class MOZ_NON_PARAM Val : public LitVal { + public: + Val() : LitVal() {} + explicit Val(ValType type) : LitVal(type) {} + explicit Val(const LitVal& val); + explicit Val(uint32_t i32) : LitVal(i32) {} + explicit Val(uint64_t i64) : LitVal(i64) {} + explicit Val(float f32) : LitVal(f32) {} + explicit Val(double f64) : LitVal(f64) {} + explicit Val(V128 v128) : LitVal(v128) {} + explicit Val(ValType type, AnyRef val) : LitVal(type, AnyRef::null()) { + MOZ_ASSERT(type.isRefRepr()); + cell_.ref_ = val; + } + explicit Val(ValType type, FuncRef val) : LitVal(type, AnyRef::null()) { + MOZ_ASSERT(type.refType().isFuncHierarchy()); + cell_.ref_ = val.asAnyRef(); + } + + Val(const Val&) = default; + Val& operator=(const Val&) = default; + + bool operator==(const Val& rhs) const { + if (type_ != rhs.type_) { + return false; + } + switch (type_.kind()) { + case ValType::I32: + return cell_.i32_ == rhs.cell_.i32_; + case ValType::I64: + return cell_.i64_ == rhs.cell_.i64_; + case ValType::F32: + return cell_.f32_ == rhs.cell_.f32_; + case ValType::F64: + return cell_.f64_ == rhs.cell_.f64_; + case ValType::V128: + return cell_.v128_ == rhs.cell_.v128_; + case ValType::Ref: + return cell_.ref_ == rhs.cell_.ref_; + } + MOZ_ASSERT_UNREACHABLE(); + return false; + } + bool operator!=(const Val& rhs) const { return !(*this == rhs); } + + bool isJSObject() const { + return type_.isValid() && type_.isRefRepr() && !cell_.ref_.isNull(); + } + + JSObject* asJSObject() const { + MOZ_ASSERT(isJSObject()); + return cell_.ref_.asJSObject(); + } + + JSObject** asJSObjectAddress() const { + return cell_.ref_.asJSObjectAddress(); + } + + // Read from `loc` which is a rooted location and needs no barriers. + void readFromRootedLocation(const void* loc); + + // Initialize from `loc` which is a rooted location and needs no barriers. + void initFromRootedLocation(ValType type, const void* loc); + void initFromHeapLocation(ValType type, const void* loc); + + // Write to `loc` which is a rooted location and needs no barriers. + void writeToRootedLocation(void* loc, bool mustWrite64) const; + + // Read from `loc` which is in the heap. + void readFromHeapLocation(const void* loc); + // Write to `loc` which is in the heap and must be barriered. + void writeToHeapLocation(void* loc) const; + + // See the comment for `ToWebAssemblyValue` below. + static bool fromJSValue(JSContext* cx, ValType targetType, HandleValue val, + MutableHandle rval); + // See the comment for `ToJSValue` below. + bool toJSValue(JSContext* cx, MutableHandleValue rval) const; + + void trace(JSTracer* trc) const; +}; + +using GCPtrVal = GCPtr; +using RootedVal = Rooted; +using HandleVal = Handle; +using MutableHandleVal = MutableHandle; + +using ValVector = GCVector; +using RootedValVector = Rooted; +using HandleValVector = Handle; +using MutableHandleValVector = MutableHandle; + +template +using ValVectorN = GCVector; +template +using RootedValVectorN = Rooted>; + +// Check a value against the given reference type. If the targetType +// is RefType::Extern then the test always passes, but the value may be boxed. +// If the test passes then the value is stored either in fnval (for +// RefType::Func) or in refval (for other types); this split is not strictly +// necessary but is convenient for the users of this function. +// +// This can return false if the type check fails, or if a boxing into AnyRef +// throws an OOM. +[[nodiscard]] extern bool CheckRefType(JSContext* cx, RefType targetType, + HandleValue v, + MutableHandleFunction fnval, + MutableHandleAnyRef refval); + +// The same as above for when the target type is 'funcref'. +[[nodiscard]] extern bool CheckFuncRefValue(JSContext* cx, HandleValue v, + MutableHandleFunction fun); + +// The same as above for when the target type is 'anyref'. +[[nodiscard]] extern bool CheckAnyRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp); + +// The same as above for when the target type is 'nullexternref'. +[[nodiscard]] extern bool CheckNullExternRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp); + +// The same as above for when the target type is 'nullfuncref'. +[[nodiscard]] extern bool CheckNullFuncRefValue(JSContext* cx, HandleValue v, + MutableHandleFunction fun); + +// The same as above for when the target type is 'nullref'. +[[nodiscard]] extern bool CheckNullRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp); + +// The same as above for when the target type is 'eqref'. +[[nodiscard]] extern bool CheckEqRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp); + +// The same as above for when the target type is 'structref'. +[[nodiscard]] extern bool CheckStructRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp); + +// The same as above for when the target type is 'arrayref'. +[[nodiscard]] extern bool CheckArrayRefValue(JSContext* cx, HandleValue v, + MutableHandleAnyRef vp); + +// The same as above for when the target type is '(ref T)'. +[[nodiscard]] extern bool CheckTypeRefValue(JSContext* cx, + const TypeDef* typeDef, + HandleValue v, + MutableHandleAnyRef vp); +class NoDebug; +class DebugCodegenVal; + +// The level of coercion to apply in `ToWebAssemblyValue` and `ToJSValue`. +enum class CoercionLevel { + // The default coercions given by the JS-API specification. + Spec, + // Allow for the coercions given by `Spec` but also use WebAssembly.Global + // as a container for lossless conversions. This is only available through + // the wasmLosslessInvoke testing function and is used in tests. + Lossless, +}; + +// Coercion function from a JS value to a WebAssembly value [1]. +// +// This function may fail for any of the following reasons: +// * The input value has an incorrect type for the targetType +// * The targetType is not exposable +// * An OOM ocurred +// An error will be set upon failure. +// +// [1] https://webassembly.github.io/spec/js-api/index.html#towebassemblyvalue +template +extern bool ToWebAssemblyValue(JSContext* cx, HandleValue val, FieldType type, + void* loc, bool mustWrite64, + CoercionLevel level = CoercionLevel::Spec); +template +extern bool ToWebAssemblyValue(JSContext* cx, HandleValue val, ValType type, + void* loc, bool mustWrite64, + CoercionLevel level = CoercionLevel::Spec); + +// Coercion function from a WebAssembly value to a JS value [1]. +// +// This function will only fail if an OOM ocurred. If the type of WebAssembly +// value being coerced is not exposable to JS, then it will be coerced to +// 'undefined'. Callers are responsible for guarding against this if this is +// not desirable. +// +// [1] https://webassembly.github.io/spec/js-api/index.html#tojsvalue +template +extern bool ToJSValue(JSContext* cx, const void* src, FieldType type, + MutableHandleValue dst, + CoercionLevel level = CoercionLevel::Spec); +template +extern bool ToJSValueMayGC(FieldType type); +template +extern bool ToJSValue(JSContext* cx, const void* src, ValType type, + MutableHandleValue dst, + CoercionLevel level = CoercionLevel::Spec); +template +extern bool ToJSValueMayGC(ValType type); +} // namespace wasm + +template <> +struct InternalBarrierMethods { + STATIC_ASSERT_ANYREF_IS_JSOBJECT; + + static bool isMarkable(const wasm::Val& v) { return v.isJSObject(); } + + static void preBarrier(const wasm::Val& v) { + if (v.isJSObject()) { + gc::PreWriteBarrier(v.asJSObject()); + } + } + + static MOZ_ALWAYS_INLINE void postBarrier(wasm::Val* vp, + const wasm::Val& prev, + const wasm::Val& next) { + MOZ_RELEASE_ASSERT(!prev.type().isValid() || prev.type() == next.type()); + JSObject* prevObj = prev.isJSObject() ? prev.asJSObject() : nullptr; + JSObject* nextObj = next.isJSObject() ? next.asJSObject() : nullptr; + if (nextObj) { + JSObject::postWriteBarrier(vp->asJSObjectAddress(), prevObj, nextObj); + } + } + + static void readBarrier(const wasm::Val& v) { + if (v.isJSObject()) { + gc::ReadBarrier(v.asJSObject()); + } + } + +#ifdef DEBUG + static void assertThingIsNotGray(const wasm::Val& v) { + if (v.isJSObject()) { + JS::AssertObjectIsNotGray(v.asJSObject()); + } + } +#endif +}; + +template <> +struct InternalBarrierMethods { + STATIC_ASSERT_ANYREF_IS_JSOBJECT; + + static bool isMarkable(const wasm::AnyRef v) { return !v.isNull(); } + + static void preBarrier(const wasm::AnyRef v) { + if (!v.isNull()) { + gc::PreWriteBarrier(v.asJSObject()); + } + } + + static MOZ_ALWAYS_INLINE void postBarrier(wasm::AnyRef* vp, + const wasm::AnyRef prev, + const wasm::AnyRef next) { + JSObject* prevObj = !prev.isNull() ? prev.asJSObject() : nullptr; + JSObject* nextObj = !next.isNull() ? next.asJSObject() : nullptr; + if (nextObj) { + JSObject::postWriteBarrier(vp->asJSObjectAddress(), prevObj, nextObj); + } + } + + static void readBarrier(const wasm::AnyRef v) { + if (!v.isNull()) { + gc::ReadBarrier(v.asJSObject()); + } + } + +#ifdef DEBUG + static void assertThingIsNotGray(const wasm::AnyRef v) { + if (!v.isNull()) { + JS::AssertObjectIsNotGray(v.asJSObject()); + } + } +#endif +}; + +} // namespace js + +#endif // wasm_val_h diff --git a/js/src/wasm/moz.build b/js/src/wasm/moz.build new file mode 100644 index 0000000000..3899fbf41b --- /dev/null +++ b/js/src/wasm/moz.build @@ -0,0 +1,66 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FINAL_LIBRARY = "js" + +# Includes should be relative to parent path +LOCAL_INCLUDES += ["!..", ".."] + +include("../js-config.mozbuild") +include("../js-cxxflags.mozbuild") + +UNIFIED_SOURCES += [ + "AsmJS.cpp", + "WasmBaselineCompile.cpp", + "WasmBCFrame.cpp", + "WasmBCMemory.cpp", + "WasmBinary.cpp", + "WasmBuiltins.cpp", + "WasmCode.cpp", + "WasmCodegenTypes.cpp", + "WasmCompile.cpp", + "WasmDebug.cpp", + "WasmDebugFrame.cpp", + "WasmFrameIter.cpp", + "WasmGC.cpp", + "WasmGcObject.cpp", + "WasmGenerator.cpp", + "WasmInitExpr.cpp", + "WasmInstance.cpp", + "WasmIntrinsic.cpp", + "WasmIonCompile.cpp", + "WasmJS.cpp", + "WasmLog.cpp", + "WasmMemory.cpp", + "WasmModule.cpp", + "WasmModuleTypes.cpp", + "WasmOpIter.cpp", + "WasmProcess.cpp", + "WasmRealm.cpp", + "WasmSerialize.cpp", + "WasmStubs.cpp", + "WasmTable.cpp", + "WasmTypeDef.cpp", + "WasmValidate.cpp", + "WasmValType.cpp", + "WasmValue.cpp", +] + +# Generate wasm/WasmIntrinsicGenerated.h from wasm/WasmIntrinsic.yaml +GeneratedFile( + "WasmIntrinsicGenerated.h", + script="GenerateIntrinsics.py", + inputs=["WasmIntrinsic.yaml"], +) + +# We don't support signals for wasi yet. +if CONFIG["OS_ARCH"] != "WASI": + UNIFIED_SOURCES += ["WasmSignalHandlers.cpp"] + +# Make sure all WebAssembly code is built with libfuzzer +# coverage instrumentation in FUZZING mode. +if CONFIG["FUZZING_INTERFACES"] and CONFIG["LIBFUZZER"]: + include("/tools/fuzzing/libfuzzer-config.mozbuild") -- cgit v1.2.3