1 files changed, 368 insertions, 0 deletions
diff --git a/js/src/wasm/WasmGcObject-inl.h b/js/src/wasm/WasmGcObject-inl.h
new file mode 100644
index 0000000000..17800f41f1
--- /dev/null
+++ b/js/src/wasm/WasmGcObject-inl.h
@@ -0,0 +1,368 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef wasm_WasmGcObject_inl_h
+#define wasm_WasmGcObject_inl_h
+
+#include "wasm/WasmGcObject.h"
+
+#include "mozilla/Attributes.h"
+#include "mozilla/DebugOnly.h"
+#include "util/Memory.h"
+
+#include "gc/Nursery-inl.h"
+#include "gc/ObjectKind-inl.h"
+#include "vm/JSContext-inl.h"
+
+//=========================================================================
+// WasmStructObject inlineable allocation methods
+
+namespace js {
+
+/* static */
+template <bool ZeroFields>
+MOZ_ALWAYS_INLINE WasmStructObject* WasmStructObject::createStructIL(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap) {
+  // It is up to our caller to ensure that `typeDefData` refers to a type that
+  // doesn't need OOL storage.
+
+  MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp));
+  MOZ_ASSERT(!typeDefData->clasp->isNativeObject());
+  debugCheckNewObject(typeDefData->shape, typeDefData->allocKind, initialHeap);
+
+  mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef;
+  MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct);
+
+  // This doesn't need to be rooted, since all we do with it prior to
+  // return is to zero out the fields (and then only if ZeroFields is true).
+  WasmStructObject* structObj = (WasmStructObject*)cx->newCell<WasmGcObject>(
+      typeDefData->allocKind, initialHeap, typeDefData->clasp,
+      &typeDefData->allocSite);
+  if (MOZ_UNLIKELY(!structObj)) {
+    ReportOutOfMemory(cx);
+    return nullptr;
+  }
+
+  MOZ_ASSERT((uintptr_t(structObj->inlineData()) % sizeof(uintptr_t)) == 0);
+  structObj->initShape(typeDefData->shape);
+  structObj->superTypeVector_ = typeDefData->superTypeVector;
+  structObj->outlineData_ = nullptr;
+  if constexpr (ZeroFields) {
+    uint32_t totalBytes = typeDefData->structTypeSize;
+    MOZ_ASSERT(totalBytes == typeDef->structType().size_);
+    MOZ_ASSERT(totalBytes <= WasmStructObject_MaxInlineBytes);
+    MOZ_ASSERT((totalBytes % sizeof(uintptr_t)) == 0);
+    memset(structObj->inlineData(), 0, totalBytes);
+  }
+
+  js::gc::gcprobes::CreateObject(structObj);
+  probes::CreateObject(cx, structObj);
+
+  return structObj;
+}
+
+/* static */
+template <bool ZeroFields>
+MOZ_ALWAYS_INLINE WasmStructObject* WasmStructObject::createStructOOL(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap) {
+  // It is up to our caller to ensure that `typeDefData` refers to a type that
+  // needs OOL storage.
+
+  MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp));
+  MOZ_ASSERT(!typeDefData->clasp->isNativeObject());
+  debugCheckNewObject(typeDefData->shape, typeDefData->allocKind, initialHeap);
+
+  mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef;
+  MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Struct);
+
+  uint32_t totalBytes = typeDefData->structTypeSize;
+  MOZ_ASSERT(totalBytes == typeDef->structType().size_);
+  MOZ_ASSERT(totalBytes > WasmStructObject_MaxInlineBytes);
+  MOZ_ASSERT((totalBytes % sizeof(uintptr_t)) == 0);
+
+  uint32_t inlineBytes, outlineBytes;
+  WasmStructObject::getDataByteSizes(totalBytes, &inlineBytes, &outlineBytes);
+  MOZ_ASSERT(inlineBytes == WasmStructObject_MaxInlineBytes);
+  MOZ_ASSERT(outlineBytes > 0);
+
+  // Allocate the outline data area before allocating the object so that we can
+  // infallibly initialize the outline data area.
+  Nursery& nursery = cx->nursery();
+  PointerAndUint7 outlineData =
+      nursery.mallocedBlockCache().alloc(outlineBytes);
+  if (MOZ_UNLIKELY(!outlineData.pointer())) {
+    ReportOutOfMemory(cx);
+    return nullptr;
+  }
+
+  // See corresponding comment in WasmArrayObject::createArray.
+  Rooted<WasmStructObject*> structObj(cx);
+  structObj = (WasmStructObject*)cx->newCell<WasmGcObject>(
+      typeDefData->allocKind, initialHeap, typeDefData->clasp,
+      &typeDefData->allocSite);
+  if (MOZ_UNLIKELY(!structObj)) {
+    ReportOutOfMemory(cx);
+    if (outlineData.pointer()) {
+      nursery.mallocedBlockCache().free(outlineData);
+    }
+    return nullptr;
+  }
+
+  MOZ_ASSERT((uintptr_t(structObj->inlineData()) % sizeof(uintptr_t)) == 0);
+  structObj->initShape(typeDefData->shape);
+  structObj->superTypeVector_ = typeDefData->superTypeVector;
+
+  // Initialize the outline data fields
+  structObj->outlineData_ = (uint8_t*)outlineData.pointer();
+  if constexpr (ZeroFields) {
+    memset(structObj->inlineData(), 0, inlineBytes);
+    memset(outlineData.pointer(), 0, outlineBytes);
+  }
+
+  if (MOZ_LIKELY(js::gc::IsInsideNursery(structObj))) {
+    // See corresponding comment in WasmArrayObject::createArrayNonEmpty.
+    if (MOZ_UNLIKELY(!nursery.registerTrailer(outlineData, outlineBytes))) {
+      nursery.mallocedBlockCache().free(outlineData);
+      ReportOutOfMemory(cx);
+      return nullptr;
+    }
+  } else {
+    // See corresponding comment in WasmArrayObject::createArrayNonEmpty.
+    MOZ_ASSERT(structObj->isTenured());
+    AddCellMemory(structObj, outlineBytes + wasm::TrailerBlockOverhead,
+                  MemoryUse::WasmTrailerBlock);
+  }
+
+  js::gc::gcprobes::CreateObject(structObj);
+  probes::CreateObject(cx, structObj);
+
+  return structObj;
+}
+
+//=========================================================================
+// WasmArrayObject inlineable allocation methods
+
+/* static */
+inline gc::AllocKind WasmArrayObject::allocKindForOOL() {
+  gc::AllocKind allocKind =
+      gc::GetGCObjectKindForBytes(sizeof(WasmArrayObject));
+  if (CanChangeToBackgroundAllocKind(allocKind, &WasmArrayObject::class_)) {
+    allocKind = ForegroundToBackgroundAllocKind(allocKind);
+  }
+  return allocKind;
+}
+
+/* static */
+inline gc::AllocKind WasmArrayObject::allocKindForIL(uint32_t storageBytes) {
+  gc::AllocKind allocKind =
+      gc::GetGCObjectKindForBytes(sizeof(WasmArrayObject) + storageBytes);
+  if (CanChangeToBackgroundAllocKind(allocKind, &WasmArrayObject::class_)) {
+    allocKind = ForegroundToBackgroundAllocKind(allocKind);
+  }
+  return allocKind;
+}
+
+inline gc::AllocKind WasmArrayObject::allocKind() const {
+  if (isDataInline()) {
+    uint32_t storageBytes = calcStorageBytes(
+        typeDef().arrayType().elementType_.size(), numElements_);
+    return allocKindForIL(storageBytes);
+  }
+
+  return allocKindForOOL();
+}
+
+/* static */
+template <bool ZeroFields>
+MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArrayOOL(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements, uint32_t storageBytes) {
+  STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32;
+
+  MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp));
+  MOZ_ASSERT(!typeDefData->clasp->isNativeObject());
+  MOZ_ASSERT(typeDefData->allocKind == gc::AllocKind::INVALID);
+  gc::AllocKind allocKind = allocKindForOOL();
+  debugCheckNewObject(typeDefData->shape, allocKind, initialHeap);
+
+  mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef;
+  MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Array);
+
+  // This routine is for large arrays with out-of-line data only. For small
+  // arrays use createArrayIL.
+  MOZ_ASSERT(storageBytes > WasmArrayObject_MaxInlineBytes);
+
+  // Allocate the outline data before allocating the object so that we can
+  // infallibly initialize the pointer on the array object after it is
+  // allocated.
+  Nursery& nursery = cx->nursery();
+  PointerAndUint7 outlineAlloc(nullptr, 0);
+  outlineAlloc = nursery.mallocedBlockCache().alloc(storageBytes);
+  if (MOZ_UNLIKELY(!outlineAlloc.pointer())) {
+    ReportOutOfMemory(cx);
+    return nullptr;
+  }
+
+  // It's unfortunate that `arrayObj` has to be rooted, since this is a hot
+  // path and rooting costs around 15 instructions.  It is the call to
+  // registerTrailer that makes it necessary.
+  Rooted<WasmArrayObject*> arrayObj(cx);
+  arrayObj = (WasmArrayObject*)cx->newCell<WasmGcObject>(
+      allocKind, initialHeap, typeDefData->clasp, &typeDefData->allocSite);
+  if (MOZ_UNLIKELY(!arrayObj)) {
+    ReportOutOfMemory(cx);
+    if (outlineAlloc.pointer()) {
+      nursery.mallocedBlockCache().free(outlineAlloc);
+    }
+    return nullptr;
+  }
+
+  DataHeader* outlineHeader = (DataHeader*)outlineAlloc.pointer();
+  uint8_t* outlineData = (uint8_t*)(outlineHeader + 1);
+  *outlineHeader = DataIsOOL;
+
+  arrayObj->initShape(typeDefData->shape);
+  arrayObj->superTypeVector_ = typeDefData->superTypeVector;
+  arrayObj->numElements_ = numElements;
+  arrayObj->data_ = outlineData;
+  if constexpr (ZeroFields) {
+    uint32_t dataBytes = storageBytes - sizeof(DataHeader);
+    MOZ_ASSERT(dataBytes >= numElements * typeDefData->arrayElemSize);
+    memset(arrayObj->data_, 0, dataBytes);
+  }
+
+  MOZ_ASSERT(!arrayObj->isDataInline());
+
+  if (MOZ_LIKELY(js::gc::IsInsideNursery(arrayObj))) {
+    // We need to register the OOL area with the nursery, so it will be freed
+    // after GCing of the nursery if `arrayObj_` doesn't make it into the
+    // tenured heap.  Note, the nursery will keep a running total of the
+    // current trailer block sizes, so it can decide to do a (minor)
+    // collection if that becomes excessive.
+    if (MOZ_UNLIKELY(!nursery.registerTrailer(outlineAlloc, storageBytes))) {
+      nursery.mallocedBlockCache().free(outlineAlloc);
+      ReportOutOfMemory(cx);
+      return nullptr;
+    }
+  } else {
+    MOZ_ASSERT(arrayObj->isTenured());
+    // Register the trailer size with the major GC mechanism, so that can also
+    // is able to decide if that space use warrants a (major) collection.
+    AddCellMemory(arrayObj, storageBytes + wasm::TrailerBlockOverhead,
+                  MemoryUse::WasmTrailerBlock);
+  }
+
+  js::gc::gcprobes::CreateObject(arrayObj);
+  probes::CreateObject(cx, arrayObj);
+
+  return arrayObj;
+}
+
+template WasmArrayObject* WasmArrayObject::createArrayOOL<true>(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements, uint32_t storageBytes);
+template WasmArrayObject* WasmArrayObject::createArrayOOL<false>(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements, uint32_t storageBytes);
+
+/* static */
+template <bool ZeroFields>
+MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArrayIL(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements, uint32_t storageBytes) {
+  STATIC_ASSERT_WASMARRAYELEMENTS_NUMELEMENTS_IS_U32;
+
+  MOZ_ASSERT(IsWasmGcObjectClass(typeDefData->clasp));
+  MOZ_ASSERT(!typeDefData->clasp->isNativeObject());
+  MOZ_ASSERT(typeDefData->allocKind == gc::AllocKind::INVALID);
+  gc::AllocKind allocKind = allocKindForIL(storageBytes);
+  debugCheckNewObject(typeDefData->shape, allocKind, initialHeap);
+
+  mozilla::DebugOnly<const wasm::TypeDef*> typeDef = typeDefData->typeDef;
+  MOZ_ASSERT(typeDef->kind() == wasm::TypeDefKind::Array);
+
+  MOZ_ASSERT(storageBytes <= WasmArrayObject_MaxInlineBytes);
+
+  // There's no need for `arrayObj` to be rooted, since the only thing we're
+  // going to do is fill in some bits of it, then return it.
+  WasmArrayObject* arrayObj = (WasmArrayObject*)cx->newCell<WasmGcObject>(
+      allocKind, initialHeap, typeDefData->clasp, &typeDefData->allocSite);
+  if (MOZ_UNLIKELY(!arrayObj)) {
+    ReportOutOfMemory(cx);
+    return nullptr;
+  }
+
+  arrayObj->initShape(typeDefData->shape);
+  arrayObj->superTypeVector_ = typeDefData->superTypeVector;
+  arrayObj->numElements_ = numElements;
+
+  DataHeader* inlineHeader =
+      WasmArrayObject::addressOfInlineDataHeader(arrayObj);
+  uint8_t* inlineData = WasmArrayObject::addressOfInlineData(arrayObj);
+  *inlineHeader = DataIsIL;
+  arrayObj->data_ = inlineData;
+
+  if constexpr (ZeroFields) {
+    uint32_t dataBytes = storageBytes - sizeof(DataHeader);
+    MOZ_ASSERT(dataBytes >= numElements * typeDefData->arrayElemSize);
+
+    if (numElements > 0) {
+      memset(arrayObj->data_, 0, dataBytes);
+    }
+  }
+
+  MOZ_ASSERT(arrayObj->isDataInline());
+
+  js::gc::gcprobes::CreateObject(arrayObj);
+  probes::CreateObject(cx, arrayObj);
+
+  return arrayObj;
+}
+
+template WasmArrayObject* WasmArrayObject::createArrayIL<true>(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements, uint32_t storageBytes);
+template WasmArrayObject* WasmArrayObject::createArrayIL<false>(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements, uint32_t storageBytes);
+
+/* static */
+template <bool ZeroFields>
+MOZ_ALWAYS_INLINE WasmArrayObject* WasmArrayObject::createArray(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements) {
+  MOZ_ASSERT(typeDefData->arrayElemSize ==
+             typeDefData->typeDef->arrayType().elementType_.size());
+  CheckedUint32 storageBytes =
+      calcStorageBytesChecked(typeDefData->arrayElemSize, numElements);
+  if (!storageBytes.isValid() ||
+      storageBytes.value() > uint32_t(wasm::MaxArrayPayloadBytes)) {
+    JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr,
+                             JSMSG_WASM_ARRAY_IMP_LIMIT);
+    return nullptr;
+  }
+
+  if (storageBytes.value() <= WasmArrayObject_MaxInlineBytes) {
+    return createArrayIL<ZeroFields>(cx, typeDefData, initialHeap, numElements,
+                                     storageBytes.value());
+  }
+
+  return createArrayOOL<ZeroFields>(cx, typeDefData, initialHeap, numElements,
+                                    storageBytes.value());
+}
+
+template WasmArrayObject* WasmArrayObject::createArray<true>(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements);
+template WasmArrayObject* WasmArrayObject::createArray<false>(
+    JSContext* cx, wasm::TypeDefInstanceData* typeDefData,
+    js::gc::Heap initialHeap, uint32_t numElements);
+
+}  // namespace js
+
+#endif /* wasm_WasmGcObject_inl_h */