diff options
Diffstat (limited to 'fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems')
5 files changed, 1656 insertions, 0 deletions
diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_alloc.c b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_alloc.c new file mode 100644 index 000000000..5c2a628a2 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_alloc.c @@ -0,0 +1,794 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" + +static inline bool +hmu_is_in_heap(void *hmu, gc_uint8 *heap_base_addr, gc_uint8 *heap_end_addr) +{ + gc_uint8 *addr = (gc_uint8 *)hmu; + return (addr >= heap_base_addr && addr < heap_end_addr) ? true : false; +} + +/** + * Remove a node from the tree it belongs to + * + * @param p the node to remove, can not be NULL, can not be the ROOT node + * the node will be removed from the tree, and the left, right and + * parent pointers of the node @p will be set to be NULL. Other fields + * won't be touched. The tree will be re-organized so that the order + * conditions are still satisified. + */ +static bool +remove_tree_node(gc_heap_t *heap, hmu_tree_node_t *p) +{ + hmu_tree_node_t *q = NULL, **slot = NULL, *parent; + hmu_tree_node_t *root = heap->kfc_tree_root; + gc_uint8 *base_addr = heap->base_addr; + gc_uint8 *end_addr = base_addr + heap->current_size; + + bh_assert(p); + + parent = p->parent; + if (!parent || p == root /* p can not be the ROOT node */ + || !hmu_is_in_heap(p, base_addr, end_addr) + || (parent != root && !hmu_is_in_heap(parent, base_addr, end_addr))) { + goto fail; + } + + /* get the slot which holds pointer to node p */ + if (p == p->parent->right) { + /* Don't use `slot = &p->parent->right` to avoid compiler warning */ + slot = (hmu_tree_node_t **)((uint8 *)p->parent + + offsetof(hmu_tree_node_t, right)); + } + else if (p == p->parent->left) { + /* p should be a child of its parent */ + /* Don't use `slot = &p->parent->left` to avoid compiler warning */ + slot = (hmu_tree_node_t **)((uint8 *)p->parent + + offsetof(hmu_tree_node_t, left)); + } + else { + goto fail; + } + + /** + * algorithms used to remove node p + * case 1: if p has no left child, replace p with its right child + * case 2: if p has no right child, replace p with its left child + * case 3: otherwise, find p's predecessor, remove it from the tree + * and replace p with it. + * use predecessor can keep the left <= root < right condition. + */ + + if (!p->left) { + /* move right child up*/ + *slot = p->right; + if (p->right) { + if (!hmu_is_in_heap(p->right, base_addr, end_addr)) { + goto fail; + } + p->right->parent = p->parent; + } + + p->left = p->right = p->parent = NULL; + return true; + } + + if (!p->right) { + /* move left child up*/ + *slot = p->left; + if (!hmu_is_in_heap(p->left, base_addr, end_addr)) { + goto fail; + } + /* p->left can never be NULL unless it is corrupted. */ + p->left->parent = p->parent; + + p->left = p->right = p->parent = NULL; + return true; + } + + /* both left & right exist, find p's predecessor at first*/ + q = p->left; + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } + while (q->right) { + q = q->right; + if (!hmu_is_in_heap(q, base_addr, end_addr)) { + goto fail; + } + } + + /* remove from the tree*/ + if (!remove_tree_node(heap, q)) + return false; + + *slot = q; + q->parent = p->parent; + q->left = p->left; + q->right = p->right; + if (q->left) { + if (!hmu_is_in_heap(q->left, base_addr, end_addr)) { + goto fail; + } + q->left->parent = q; + } + if (q->right) { + if (!hmu_is_in_heap(q->right, base_addr, end_addr)) { + goto fail; + } + q->right->parent = q; + } + + p->left = p->right = p->parent = NULL; + + return true; +fail: + heap->is_heap_corrupted = true; + return false; +} + +static bool +unlink_hmu(gc_heap_t *heap, hmu_t *hmu) +{ + gc_uint8 *base_addr, *end_addr; + gc_size_t size; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(hmu && (gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + + if (hmu_get_ut(hmu) != HMU_FC) { + heap->is_heap_corrupted = true; + return false; + } + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + size = hmu_get_size(hmu); + + if (HMU_IS_FC_NORMAL(size)) { + uint32 node_idx = size >> 3; + hmu_normal_node_t *node_prev = NULL, *node_next; + hmu_normal_node_t *node = heap->kfc_normal_list[node_idx].next; + + while (node) { + if (!hmu_is_in_heap(node, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } + node_next = get_hmu_normal_node_next(node); + if ((hmu_t *)node == hmu) { + if (!node_prev) /* list head */ + heap->kfc_normal_list[node_idx].next = node_next; + else + set_hmu_normal_node_next(node_prev, node_next); + break; + } + node_prev = node; + node = node_next; + } + + if (!node) { + os_printf("[GC_ERROR]couldn't find the node in the normal list\n"); + } + } + else { + if (!remove_tree_node(heap, (hmu_tree_node_t *)hmu)) + return false; + } + return true; +} + +static void +hmu_set_free_size(hmu_t *hmu) +{ + gc_size_t size; + bh_assert(hmu && hmu_get_ut(hmu) == HMU_FC); + + size = hmu_get_size(hmu); + *((uint32 *)((char *)hmu + size) - 1) = size; +} + +/** + * Add free chunk back to KFC + * + * @param heap should not be NULL and it should be a valid heap + * @param hmu should not be NULL and it should be a HMU of length @size inside + * @heap hmu should be 8-bytes aligned + * @param size should be positive and multiple of 8 + * hmu with size @size will be added into KFC as a new FC. + */ +bool +gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size) +{ + gc_uint8 *base_addr, *end_addr; + hmu_normal_node_t *np = NULL; + hmu_tree_node_t *root = NULL, *tp = NULL, *node = NULL; + uint32 node_idx; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(hmu && (gc_uint8 *)hmu >= heap->base_addr + && (gc_uint8 *)hmu < heap->base_addr + heap->current_size); + bh_assert(((gc_uint32)(uintptr_t)hmu_to_obj(hmu) & 7) == 0); + bh_assert(size > 0 + && ((gc_uint8 *)hmu) + size + <= heap->base_addr + heap->current_size); + bh_assert(!(size & 7)); + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + hmu_set_ut(hmu, HMU_FC); + hmu_set_size(hmu, size); + hmu_set_free_size(hmu); + + if (HMU_IS_FC_NORMAL(size)) { + np = (hmu_normal_node_t *)hmu; + if (!hmu_is_in_heap(np, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } + + node_idx = size >> 3; + set_hmu_normal_node_next(np, heap->kfc_normal_list[node_idx].next); + heap->kfc_normal_list[node_idx].next = np; + return true; + } + + /* big block */ + node = (hmu_tree_node_t *)hmu; + node->size = size; + node->left = node->right = node->parent = NULL; + + /* find proper node to link this new node to */ + root = heap->kfc_tree_root; + tp = root; + bh_assert(tp->size < size); + while (1) { + if (tp->size < size) { + if (!tp->right) { + tp->right = node; + node->parent = tp; + break; + } + tp = tp->right; + } + else { /* tp->size >= size */ + if (!tp->left) { + tp->left = node; + node->parent = tp; + break; + } + tp = tp->left; + } + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return false; + } + } + return true; +} + +/** + * Find a proper hmu for required memory size + * + * @param heap should not be NULL and should be a valid heap + * @param size should cover the header and should be 8 bytes aligned + * GC will not be performed here. + * Heap extension will not be performed here. + * + * @return hmu allocated if success, which will be aligned to 8 bytes, + * NULL otherwise + */ +static hmu_t * +alloc_hmu(gc_heap_t *heap, gc_size_t size) +{ + gc_uint8 *base_addr, *end_addr; + hmu_normal_list_t *normal_head = NULL; + hmu_normal_node_t *p = NULL; + uint32 node_idx = 0, init_node_idx = 0; + hmu_tree_node_t *root = NULL, *tp = NULL, *last_tp = NULL; + hmu_t *next, *rest; + uintptr_t tp_ret; + + bh_assert(gci_is_heap_valid(heap)); + bh_assert(size > 0 && !(size & 7)); + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + if (size < GC_SMALLEST_SIZE) + size = GC_SMALLEST_SIZE; + + /* check normal list at first*/ + if (HMU_IS_FC_NORMAL(size)) { + /* find a non-empty slot in normal_node_list with good size*/ + init_node_idx = (size >> 3); + for (node_idx = init_node_idx; node_idx < HMU_NORMAL_NODE_CNT; + node_idx++) { + normal_head = heap->kfc_normal_list + node_idx; + if (normal_head->next) + break; + normal_head = NULL; + } + + /* found in normal list*/ + if (normal_head) { + bh_assert(node_idx >= init_node_idx); + + p = normal_head->next; + if (!hmu_is_in_heap(p, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } + normal_head->next = get_hmu_normal_node_next(p); + if (((gc_int32)(uintptr_t)hmu_to_obj(p) & 7) != 0) { + heap->is_heap_corrupted = true; + return NULL; + } + + if ((gc_size_t)node_idx != (uint32)init_node_idx + /* with bigger size*/ + && ((gc_size_t)node_idx << 3) >= size + GC_SMALLEST_SIZE) { + rest = (hmu_t *)(((char *)p) + size); + if (!gci_add_fc(heap, rest, (node_idx << 3) - size)) { + return NULL; + } + hmu_mark_pinuse(rest); + } + else { + size = node_idx << 3; + next = (hmu_t *)((char *)p + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) + hmu_mark_pinuse(next); + } + + heap->total_free_size -= size; + if ((heap->current_size - heap->total_free_size) + > heap->highmark_size) + heap->highmark_size = + heap->current_size - heap->total_free_size; + + hmu_set_size((hmu_t *)p, size); + return (hmu_t *)p; + } + } + + /* need to find a node in tree*/ + root = heap->kfc_tree_root; + + /* find the best node*/ + bh_assert(root); + tp = root->right; + while (tp) { + if (!hmu_is_in_heap(tp, base_addr, end_addr)) { + heap->is_heap_corrupted = true; + return NULL; + } + + if (tp->size < size) { + tp = tp->right; + continue; + } + + /* record the last node with size equal to or bigger than given size*/ + last_tp = tp; + tp = tp->left; + } + + if (last_tp) { + bh_assert(last_tp->size >= size); + + /* alloc in last_p*/ + + /* remove node last_p from tree*/ + if (!remove_tree_node(heap, last_tp)) + return NULL; + + if (last_tp->size >= size + GC_SMALLEST_SIZE) { + rest = (hmu_t *)((char *)last_tp + size); + if (!gci_add_fc(heap, rest, last_tp->size - size)) + return NULL; + hmu_mark_pinuse(rest); + } + else { + size = last_tp->size; + next = (hmu_t *)((char *)last_tp + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) + hmu_mark_pinuse(next); + } + + heap->total_free_size -= size; + if ((heap->current_size - heap->total_free_size) > heap->highmark_size) + heap->highmark_size = heap->current_size - heap->total_free_size; + + hmu_set_size((hmu_t *)last_tp, size); + tp_ret = (uintptr_t)last_tp; + return (hmu_t *)tp_ret; + } + + return NULL; +} + +/** + * Find a proper HMU with given size + * + * @param heap should not be NULL and should be a valid heap + * @param size should cover the header and should be 8 bytes aligned + * + * Note: This function will try several ways to satisfy the allocation request: + * 1. Find a proper on available HMUs. + * 2. GC will be triggered if 1 failed. + * 3. Find a proper on available HMUS. + * 4. Return NULL if 3 failed + * + * @return hmu allocated if success, which will be aligned to 8 bytes, + * NULL otherwise + */ +static hmu_t * +alloc_hmu_ex(gc_heap_t *heap, gc_size_t size) +{ + bh_assert(gci_is_heap_valid(heap)); + bh_assert(size > 0 && !(size & 7)); + + return alloc_hmu(heap, size); +} + +static unsigned long g_total_malloc = 0; +static unsigned long g_total_free = 0; + +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_alloc_vo(void *vheap, gc_size_t size) +#else +gc_object_t +gc_alloc_vo_internal(void *vheap, gc_size_t size, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL; + gc_object_t ret = (gc_object_t)NULL; + gc_size_t tot_size = 0, tot_size_unaligned; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } + + os_mutex_lock(&heap->lock); + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + + g_total_malloc += tot_size; + + hmu_set_ut(hmu, HMU_VO); + hmu_unfree_vo(hmu); + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + if (tot_size > tot_size_unaligned) + /* clear buffer appended by GC_ALIGN_8() */ + memset((uint8 *)ret + size, 0, tot_size - tot_size_unaligned); + +finish: + os_mutex_unlock(&heap->lock); + return ret; +} + +#if BH_ENABLE_GC_VERIFY == 0 +gc_object_t +gc_realloc_vo(void *vheap, void *ptr, gc_size_t size) +#else +gc_object_t +gc_realloc_vo_internal(void *vheap, void *ptr, gc_size_t size, const char *file, + int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + hmu_t *hmu = NULL, *hmu_old = NULL, *hmu_next; + gc_object_t ret = (gc_object_t)NULL, obj_old = (gc_object_t)ptr; + gc_size_t tot_size, tot_size_unaligned, tot_size_old = 0, tot_size_next; + gc_size_t obj_size, obj_size_old; + gc_uint8 *base_addr, *end_addr; + hmu_type_t ut; + + /* hmu header + prefix + obj + suffix */ + tot_size_unaligned = HMU_SIZE + OBJ_PREFIX_SIZE + size + OBJ_SUFFIX_SIZE; + /* aligned size*/ + tot_size = GC_ALIGN_8(tot_size_unaligned); + if (tot_size < size) + /* integer overflow */ + return NULL; + + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, allocate memory failed.\n"); + return NULL; + } + + if (obj_old) { + hmu_old = obj_to_hmu(obj_old); + tot_size_old = hmu_get_size(hmu_old); + if (tot_size <= tot_size_old) + /* current node alreay meets requirement */ + return obj_old; + } + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + os_mutex_lock(&heap->lock); + + if (hmu_old) { + hmu_next = (hmu_t *)((char *)hmu_old + tot_size_old); + if (hmu_is_in_heap(hmu_next, base_addr, end_addr)) { + ut = hmu_get_ut(hmu_next); + tot_size_next = hmu_get_size(hmu_next); + if (ut == HMU_FC && tot_size <= tot_size_old + tot_size_next) { + /* current node and next node meets requirement */ + if (!unlink_hmu(heap, hmu_next)) { + os_mutex_unlock(&heap->lock); + return NULL; + } + hmu_set_size(hmu_old, tot_size); + memset((char *)hmu_old + tot_size_old, 0, + tot_size - tot_size_old); +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu_old, tot_size, file, line); +#endif + if (tot_size < tot_size_old + tot_size_next) { + hmu_next = (hmu_t *)((char *)hmu_old + tot_size); + tot_size_next = tot_size_old + tot_size_next - tot_size; + if (!gci_add_fc(heap, hmu_next, tot_size_next)) { + os_mutex_unlock(&heap->lock); + return NULL; + } + } + os_mutex_unlock(&heap->lock); + return obj_old; + } + } + } + + hmu = alloc_hmu_ex(heap, tot_size); + if (!hmu) + goto finish; + + bh_assert(hmu_get_size(hmu) >= tot_size); + /* the total size allocated may be larger than + the required size, reset it here */ + tot_size = hmu_get_size(hmu); + g_total_malloc += tot_size; + + hmu_set_ut(hmu, HMU_VO); + hmu_unfree_vo(hmu); + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_init_prefix_and_suffix(hmu, tot_size, file, line); +#endif + + ret = hmu_to_obj(hmu); + +finish: + + if (ret) { + obj_size = tot_size - HMU_SIZE - OBJ_PREFIX_SIZE - OBJ_SUFFIX_SIZE; + memset(ret, 0, obj_size); + if (obj_old) { + obj_size_old = + tot_size_old - HMU_SIZE - OBJ_PREFIX_SIZE - OBJ_SUFFIX_SIZE; + bh_memcpy_s(ret, obj_size, obj_old, obj_size_old); + } + } + + os_mutex_unlock(&heap->lock); + + if (ret && obj_old) + gc_free_vo(vheap, obj_old); + + return ret; +} + +/** + * Do some checking to see if given pointer is a possible valid heap + * @return GC_TRUE if all checking passed, GC_FALSE otherwise + */ +int +gci_is_heap_valid(gc_heap_t *heap) +{ + if (!heap) + return GC_FALSE; + if (heap->heap_id != (gc_handle_t)heap) + return GC_FALSE; + + return GC_TRUE; +} + +#if BH_ENABLE_GC_VERIFY == 0 +int +gc_free_vo(void *vheap, gc_object_t obj) +#else +int +gc_free_vo_internal(void *vheap, gc_object_t obj, const char *file, int line) +#endif +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + gc_uint8 *base_addr, *end_addr; + hmu_t *hmu = NULL; + hmu_t *prev = NULL; + hmu_t *next = NULL; + gc_size_t size = 0; + hmu_type_t ut; + int ret = GC_SUCCESS; + + if (!obj) { + return GC_SUCCESS; + } + + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, free memory failed.\n"); + return GC_ERROR; + } + + hmu = obj_to_hmu(obj); + + base_addr = heap->base_addr; + end_addr = base_addr + heap->current_size; + + os_mutex_lock(&heap->lock); + + if (hmu_is_in_heap(hmu, base_addr, end_addr)) { +#if BH_ENABLE_GC_VERIFY != 0 + hmu_verify(heap, hmu); +#endif + ut = hmu_get_ut(hmu); + if (ut == HMU_VO) { + if (hmu_is_vo_freed(hmu)) { + bh_assert(0); + ret = GC_ERROR; + goto out; + } + + size = hmu_get_size(hmu); + + g_total_free += size; + + heap->total_free_size += size; + + if (!hmu_get_pinuse(hmu)) { + prev = (hmu_t *)((char *)hmu - *((int *)hmu - 1)); + + if (hmu_is_in_heap(prev, base_addr, end_addr) + && hmu_get_ut(prev) == HMU_FC) { + size += hmu_get_size(prev); + hmu = prev; + if (!unlink_hmu(heap, prev)) { + ret = GC_ERROR; + goto out; + } + } + } + + next = (hmu_t *)((char *)hmu + size); + if (hmu_is_in_heap(next, base_addr, end_addr)) { + if (hmu_get_ut(next) == HMU_FC) { + size += hmu_get_size(next); + if (!unlink_hmu(heap, next)) { + ret = GC_ERROR; + goto out; + } + next = (hmu_t *)((char *)hmu + size); + } + } + + if (!gci_add_fc(heap, hmu, size)) { + ret = GC_ERROR; + goto out; + } + + if (hmu_is_in_heap(next, base_addr, end_addr)) { + hmu_unmark_pinuse(next); + } + } + else { + ret = GC_ERROR; + goto out; + } + ret = GC_SUCCESS; + goto out; + } + +out: + os_mutex_unlock(&heap->lock); + return ret; +} + +void +gc_dump_heap_stats(gc_heap_t *heap) +{ + os_printf("heap: %p, heap start: %p\n", heap, heap->base_addr); + os_printf("total free: %" PRIu32 ", current: %" PRIu32 + ", highmark: %" PRIu32 "\n", + heap->total_free_size, heap->current_size, heap->highmark_size); + os_printf("g_total_malloc=%lu, g_total_free=%lu, occupied=%lu\n", + g_total_malloc, g_total_free, g_total_malloc - g_total_free); +} + +uint32 +gc_get_heap_highmark_size(gc_heap_t *heap) +{ + return heap->highmark_size; +} + +void +gci_dump(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL; + hmu_type_t ut; + gc_size_t size; + int i = 0, p, mark; + char inuse = 'U'; + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + ut = hmu_get_ut(cur); + size = hmu_get_size(cur); + p = hmu_get_pinuse(cur); + mark = hmu_is_jo_marked(cur); + + if (ut == HMU_VO) + inuse = 'V'; + else if (ut == HMU_JO) + inuse = hmu_is_jo_marked(cur) ? 'J' : 'j'; + else if (ut == HMU_FC) + inuse = 'F'; + + if (size == 0 || size > (uint32)((uint8 *)end - (uint8 *)cur)) { + os_printf("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + return; + } + + os_printf("#%d %08" PRIx32 " %" PRIx32 " %d %d" + " %c %" PRId32 "\n", + i, (int32)((char *)cur - (char *)heap->base_addr), (int32)ut, + p, mark, inuse, (int32)hmu_obj_size(size)); +#if BH_ENABLE_GC_VERIFY != 0 + if (inuse == 'V') { + gc_object_prefix_t *prefix = (gc_object_prefix_t *)(cur + 1); + os_printf("#%s:%d\n", prefix->file_name, prefix->line_no); + } +#endif + + cur = (hmu_t *)((char *)cur + size); + i++; + } + + if (cur != end) { + os_printf("[GC_ERROR]Heap is corrupted, heap dump failed.\n"); + heap->is_heap_corrupted = true; + } +} diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_gc.h b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_gc.h new file mode 100644 index 000000000..9a74d0046 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_gc.h @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +/** + * @file ems_gc.h + * @date Wed Aug 3 10:46:38 2011 + * + * @brief This file defines GC modules types and interfaces. + */ + +#ifndef _EMS_GC_H +#define _EMS_GC_H + +#include "bh_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GC_HEAD_PADDING 4 + +#define NULL_REF ((gc_object_t)NULL) + +#define GC_SUCCESS (0) +#define GC_ERROR (-1) + +#define GC_TRUE (1) +#define GC_FALSE (0) + +#define GC_MAX_HEAP_SIZE (256 * BH_KB) + +typedef void *gc_handle_t; +typedef void *gc_object_t; +typedef int64 gc_int64; +typedef uint32 gc_uint32; +typedef int32 gc_int32; +typedef uint16 gc_uint16; +typedef int16 gc_int16; +typedef uint8 gc_uint8; +typedef int8 gc_int8; +typedef uint32 gc_size_t; + +typedef enum { + GC_STAT_TOTAL = 0, + GC_STAT_FREE, + GC_STAT_HIGHMARK, +} GC_STAT_INDEX; + +/** + * GC initialization from a buffer, which is separated into + * two parts: the beginning of the buffer is used to create + * the heap structure, and the left is used to create the + * actual pool data + * + * @param buf the buffer to be initialized to a heap + * @param buf_size the size of buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size); + +/** + * GC initialization from heap struct buffer and pool buffer + * + * @param struct_buf the struct buffer to create the heap structure + * @param struct_buf_size the size of struct buffer + * @param pool_buf the pool buffer to create pool data + * @param pool_buf_size the size of poll buffer + * + * @return gc handle if success, NULL otherwise + */ +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size); + +/** + * Destroy heap which is initilized from a buffer + * + * @param handle handle to heap needed destroy + * + * @return GC_SUCCESS if success + * GC_ERROR for bad parameters or failed system resource freeing. + */ +int +gc_destroy_with_pool(gc_handle_t handle); + +/** + * Return heap struct size + */ +uint32 +gc_get_heap_struct_size(void); + +/** + * Migrate heap from one pool buf to another pool buf + * + * @param handle handle of the new heap + * @param pool_buf_new the new pool buffer + * @param pool_buf_size the size of new pool buffer + * + * @return GC_SUCCESS if success, GC_ERROR otherwise + */ +int +gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size); + +/** + * Check whether the heap is corrupted + * + * @param handle handle of the heap + * + * @return true if success, false otherwise + */ +bool +gc_is_heap_corrupted(gc_handle_t handle); + +/** + * Get Heap Stats + * + * @param stats [out] integer array to save heap stats + * @param size [in] the size of stats + * @param mmt [in] type of heap, MMT_SHARED or MMT_INSTANCE + */ +void * +gc_heap_stats(void *heap, uint32 *stats, int size); + +#if BH_ENABLE_GC_VERIFY == 0 + +gc_object_t +gc_alloc_vo(void *heap, gc_size_t size); + +gc_object_t +gc_realloc_vo(void *heap, void *ptr, gc_size_t size); + +int +gc_free_vo(void *heap, gc_object_t obj); + +#else /* else of BH_ENABLE_GC_VERIFY */ + +gc_object_t +gc_alloc_vo_internal(void *heap, gc_size_t size, const char *file, int line); + +gc_object_t +gc_realloc_vo_internal(void *heap, void *ptr, gc_size_t size, const char *file, + int line); + +int +gc_free_vo_internal(void *heap, gc_object_t obj, const char *file, int line); + +/* clang-format off */ +#define gc_alloc_vo(heap, size) \ + gc_alloc_vo_internal(heap, size, __FILE__, __LINE__) + +#define gc_realloc_vo(heap, ptr, size) \ + gc_realloc_vo_internal(heap, ptr, size, __FILE__, __LINE__) + +#define gc_free_vo(heap, obj) \ + gc_free_vo_internal(heap, obj, __FILE__, __LINE__) +/* clang-format on */ + +#endif /* end of BH_ENABLE_GC_VERIFY */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_gc_internal.h b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_gc_internal.h new file mode 100644 index 000000000..e1ff9d61d --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_gc_internal.h @@ -0,0 +1,306 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#ifndef _EMS_GC_INTERNAL_H +#define _EMS_GC_INTERNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "bh_platform.h" +#include "ems_gc.h" + +/* HMU (heap memory unit) basic block type */ +typedef enum hmu_type_enum { + HMU_TYPE_MIN = 0, + HMU_TYPE_MAX = 3, + HMU_JO = 3, + HMU_VO = 2, + HMU_FC = 1, + HMU_FM = 0 +} hmu_type_t; + +typedef struct hmu_struct { + gc_uint32 header; +} hmu_t; + +#if BH_ENABLE_GC_VERIFY != 0 + +#if UINTPTR_MAX > UINT32_MAX +/* 2 prefix paddings for 64-bit pointer */ +#define GC_OBJECT_PREFIX_PADDING_CNT 2 +#else +/* 3 prefix paddings for 32-bit pointer */ +#define GC_OBJECT_PREFIX_PADDING_CNT 3 +#endif +#define GC_OBJECT_SUFFIX_PADDING_CNT 4 +#define GC_OBJECT_PADDING_VALUE (0x12345678) + +typedef struct gc_object_prefix { + const char *file_name; + gc_int32 line_no; + gc_int32 size; + gc_uint32 padding[GC_OBJECT_PREFIX_PADDING_CNT]; +} gc_object_prefix_t; + +typedef struct gc_object_suffix { + gc_uint32 padding[GC_OBJECT_SUFFIX_PADDING_CNT]; +} gc_object_suffix_t; + +#define OBJ_PREFIX_SIZE (sizeof(gc_object_prefix_t)) +#define OBJ_SUFFIX_SIZE (sizeof(gc_object_suffix_t)) + +void +hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, + const char *file_name, int line_no); + +void +hmu_verify(void *vheap, hmu_t *hmu); + +#define SKIP_OBJ_PREFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_PREFIX_SIZE)) +#define SKIP_OBJ_SUFFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_SUFFIX_SIZE)) + +#define OBJ_EXTRA_SIZE (HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE) + +#else /* else of BH_ENABLE_GC_VERIFY */ + +#define OBJ_PREFIX_SIZE 0 +#define OBJ_SUFFIX_SIZE 0 + +#define SKIP_OBJ_PREFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_PREFIX_SIZE)) +#define SKIP_OBJ_SUFFIX(p) ((void *)((gc_uint8 *)(p) + OBJ_SUFFIX_SIZE)) + +#define OBJ_EXTRA_SIZE (HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE) + +#endif /* end of BH_ENABLE_GC_VERIFY */ + +#define hmu_obj_size(s) ((s)-OBJ_EXTRA_SIZE) + +#define GC_ALIGN_8(s) (((uint32)(s) + 7) & (uint32)~7) + +#define GC_SMALLEST_SIZE \ + GC_ALIGN_8(HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE + 8) +#define GC_GET_REAL_SIZE(x) \ + GC_ALIGN_8(HMU_SIZE + OBJ_PREFIX_SIZE + OBJ_SUFFIX_SIZE \ + + (((x) > 8) ? (x) : 8)) + +/** + * hmu bit operation + */ + +#define SETBIT(v, offset) (v) |= ((uint32)1 << (offset)) +#define GETBIT(v, offset) ((v) & ((uint32)1 << (offset)) ? 1 : 0) +#define CLRBIT(v, offset) (v) &= (~((uint32)1 << (offset))) + +/* clang-format off */ +#define SETBITS(v, offset, size, value) \ + do { \ + (v) &= ~((((uint32)1 << size) - 1) << offset); \ + (v) |= ((uint32)value << offset); \ + } while (0) +#define CLRBITS(v, offset, size) \ + (v) &= ~((((uint32)1 << size) - 1) << offset) +#define GETBITS(v, offset, size) \ + (((v) & (((((uint32)1 << size) - 1) << offset))) >> offset) +/* clang-format on */ + +/** + * gc object layout definition + */ + +#define HMU_SIZE (sizeof(hmu_t)) + +#define hmu_to_obj(hmu) (gc_object_t)(SKIP_OBJ_PREFIX((hmu_t *)(hmu) + 1)) +#define obj_to_hmu(obj) ((hmu_t *)((gc_uint8 *)(obj)-OBJ_PREFIX_SIZE) - 1) + +#define HMU_UT_SIZE 2 +#define HMU_UT_OFFSET 30 + +/* clang-format off */ +#define hmu_get_ut(hmu) \ + GETBITS((hmu)->header, HMU_UT_OFFSET, HMU_UT_SIZE) +#define hmu_set_ut(hmu, type) \ + SETBITS((hmu)->header, HMU_UT_OFFSET, HMU_UT_SIZE, type) +#define hmu_is_ut_valid(tp) \ + (tp >= HMU_TYPE_MIN && tp <= HMU_TYPE_MAX) +/* clang-format on */ + +/* P in use bit means the previous chunk is in use */ +#define HMU_P_OFFSET 29 + +#define hmu_mark_pinuse(hmu) SETBIT((hmu)->header, HMU_P_OFFSET) +#define hmu_unmark_pinuse(hmu) CLRBIT((hmu)->header, HMU_P_OFFSET) +#define hmu_get_pinuse(hmu) GETBIT((hmu)->header, HMU_P_OFFSET) + +#define HMU_JO_VT_SIZE 27 +#define HMU_JO_VT_OFFSET 0 +#define HMU_JO_MB_OFFSET 28 + +#define hmu_mark_jo(hmu) SETBIT((hmu)->header, HMU_JO_MB_OFFSET) +#define hmu_unmark_jo(hmu) CLRBIT((hmu)->header, HMU_JO_MB_OFFSET) +#define hmu_is_jo_marked(hmu) GETBIT((hmu)->header, HMU_JO_MB_OFFSET) + +/** + * The hmu size is divisible by 8, its lowest 3 bits are 0, so we only + * store its higher bits of bit [29..3], and bit [2..0] are not stored. + * After that, the maximal heap size can be enlarged from (1<<27) = 128MB + * to (1<<27) * 8 = 1GB. + */ +#define HMU_SIZE_SIZE 27 +#define HMU_SIZE_OFFSET 0 + +#define HMU_VO_FB_OFFSET 28 + +#define hmu_is_vo_freed(hmu) GETBIT((hmu)->header, HMU_VO_FB_OFFSET) +#define hmu_unfree_vo(hmu) CLRBIT((hmu)->header, HMU_VO_FB_OFFSET) + +#define hmu_get_size(hmu) \ + (GETBITS((hmu)->header, HMU_SIZE_OFFSET, HMU_SIZE_SIZE) << 3) +#define hmu_set_size(hmu, size) \ + SETBITS((hmu)->header, HMU_SIZE_OFFSET, HMU_SIZE_SIZE, ((size) >> 3)) + +/** + * HMU free chunk management + */ + +#ifndef HMU_NORMAL_NODE_CNT +#define HMU_NORMAL_NODE_CNT 32 +#endif +#define HMU_FC_NORMAL_MAX_SIZE ((HMU_NORMAL_NODE_CNT - 1) << 3) +#define HMU_IS_FC_NORMAL(size) ((size) < HMU_FC_NORMAL_MAX_SIZE) +#if HMU_FC_NORMAL_MAX_SIZE >= GC_MAX_HEAP_SIZE +#error "Too small GC_MAX_HEAP_SIZE" +#endif + +typedef struct hmu_normal_node { + hmu_t hmu_header; + gc_int32 next_offset; +} hmu_normal_node_t; + +typedef struct hmu_normal_list { + hmu_normal_node_t *next; +} hmu_normal_list_t; + +static inline hmu_normal_node_t * +get_hmu_normal_node_next(hmu_normal_node_t *node) +{ + return node->next_offset + ? (hmu_normal_node_t *)((uint8 *)node + node->next_offset) + : NULL; +} + +static inline void +set_hmu_normal_node_next(hmu_normal_node_t *node, hmu_normal_node_t *next) +{ + if (next) { + bh_assert((uint8 *)next - (uint8 *)node < INT32_MAX); + node->next_offset = (gc_int32)(intptr_t)((uint8 *)next - (uint8 *)node); + } + else { + node->next_offset = 0; + } +} + +/** + * Define hmu_tree_node as a packed struct, since it is at the 4-byte + * aligned address and the size of hmu_head is 4, so in 64-bit target, + * the left/right/parent fields will be at 8-byte aligned address, + * we can access them directly. + */ +#if UINTPTR_MAX == UINT64_MAX +#if defined(_MSC_VER) +__pragma(pack(push, 1)); +#define __attr_packed +#elif defined(__GNUC__) || defined(__clang__) +#define __attr_packed __attribute__((packed)) +#else +#error "packed attribute isn't used to define struct hmu_tree_node" +#endif +#else /* else of UINTPTR_MAX == UINT64_MAX */ +#define __attr_packed +#endif + +typedef struct hmu_tree_node { + hmu_t hmu_header; + struct hmu_tree_node *left; + struct hmu_tree_node *right; + struct hmu_tree_node *parent; + gc_size_t size; +} __attr_packed hmu_tree_node_t; + +#if UINTPTR_MAX == UINT64_MAX +#if defined(_MSC_VER) +__pragma(pack(pop)); +#endif +#endif + +bh_static_assert(sizeof(hmu_tree_node_t) == 8 + 3 * sizeof(void *)); +bh_static_assert(offsetof(hmu_tree_node_t, left) == 4); + +#define ASSERT_TREE_NODE_ALIGNED_ACCESS(tree_node) \ + do { \ + bh_assert((((uintptr_t)&tree_node->left) & (sizeof(uintptr_t) - 1)) \ + == 0); \ + } while (0) + +typedef struct gc_heap_struct { + /* for double checking*/ + gc_handle_t heap_id; + + gc_uint8 *base_addr; + gc_size_t current_size; + + korp_mutex lock; + + hmu_normal_list_t kfc_normal_list[HMU_NORMAL_NODE_CNT]; + +#if UINTPTR_MAX == UINT64_MAX + /* make kfc_tree_root_buf 4-byte aligned and not 8-byte aligned, + so kfc_tree_root's left/right/parent fields are 8-byte aligned + and we can access them directly */ + uint32 __padding; +#endif + uint8 kfc_tree_root_buf[sizeof(hmu_tree_node_t)]; + /* point to kfc_tree_root_buf, the order in kfc_tree is: + size[left] <= size[cur] < size[right] */ + hmu_tree_node_t *kfc_tree_root; + + /* whether heap is corrupted, e.g. the hmu nodes are modified + by user */ + bool is_heap_corrupted; + + gc_size_t init_size; + gc_size_t highmark_size; + gc_size_t total_free_size; +} gc_heap_t; + +/** + * MISC internal used APIs + */ + +bool +gci_add_fc(gc_heap_t *heap, hmu_t *hmu, gc_size_t size); + +int +gci_is_heap_valid(gc_heap_t *heap); + +/** + * Verify heap integrity + */ +void +gci_verify_heap(gc_heap_t *heap); + +/** + * Dump heap nodes + */ +void +gci_dump(gc_heap_t *heap); + +#ifdef __cplusplus +} +#endif + +#endif /* end of _EMS_GC_INTERNAL_H */ diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_hmu.c b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_hmu.c new file mode 100644 index 000000000..41745e161 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_hmu.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" + +#if BH_ENABLE_GC_VERIFY != 0 + +/** + * Set default value to prefix and suffix + * @param hmu should not be NULL and should have been correctly initilized + * (except prefix and suffix part) + * @param tot_size is offered here because hmu_get_size can not be used + * till now. tot_size should not be smaller than OBJ_EXTRA_SIZE. + * For VO, tot_size should be equal to object total size. + */ +void +hmu_init_prefix_and_suffix(hmu_t *hmu, gc_size_t tot_size, + const char *file_name, int line_no) +{ + gc_object_prefix_t *prefix = NULL; + gc_object_suffix_t *suffix = NULL; + gc_uint32 i = 0; + + bh_assert(hmu); + bh_assert(hmu_get_ut(hmu) == HMU_JO || hmu_get_ut(hmu) == HMU_VO); + bh_assert(tot_size >= OBJ_EXTRA_SIZE); + bh_assert(!(tot_size & 7)); + bh_assert(hmu_get_ut(hmu) != HMU_VO || hmu_get_size(hmu) >= tot_size); + + prefix = (gc_object_prefix_t *)(hmu + 1); + suffix = + (gc_object_suffix_t *)((gc_uint8 *)hmu + tot_size - OBJ_SUFFIX_SIZE); + prefix->file_name = file_name; + prefix->line_no = line_no; + prefix->size = tot_size; + + for (i = 0; i < GC_OBJECT_PREFIX_PADDING_CNT; i++) { + prefix->padding[i] = GC_OBJECT_PADDING_VALUE; + } + + for (i = 0; i < GC_OBJECT_SUFFIX_PADDING_CNT; i++) { + suffix->padding[i] = GC_OBJECT_PADDING_VALUE; + } +} + +void +hmu_verify(void *vheap, hmu_t *hmu) +{ + gc_heap_t *heap = (gc_heap_t *)vheap; + gc_object_prefix_t *prefix = NULL; + gc_object_suffix_t *suffix = NULL; + gc_uint32 i = 0; + hmu_type_t ut; + gc_size_t size = 0; + int is_padding_ok = 1; + + bh_assert(hmu); + ut = hmu_get_ut(hmu); + bh_assert(hmu_is_ut_valid(ut)); + + prefix = (gc_object_prefix_t *)(hmu + 1); + size = prefix->size; + suffix = (gc_object_suffix_t *)((gc_uint8 *)hmu + size - OBJ_SUFFIX_SIZE); + + if (ut == HMU_VO || ut == HMU_JO) { + /* check padding*/ + for (i = 0; i < GC_OBJECT_PREFIX_PADDING_CNT; i++) { + if (prefix->padding[i] != GC_OBJECT_PADDING_VALUE) { + is_padding_ok = 0; + break; + } + } + for (i = 0; i < GC_OBJECT_SUFFIX_PADDING_CNT; i++) { + if (suffix->padding[i] != GC_OBJECT_PADDING_VALUE) { + is_padding_ok = 0; + break; + } + } + + if (!is_padding_ok) { + os_printf("Invalid padding for object created at %s:%d\n", + (prefix->file_name ? prefix->file_name : ""), + prefix->line_no); + heap->is_heap_corrupted = true; + } + } +} + +#endif /* end of BH_ENABLE_GC_VERIFY */ diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_kfc.c b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_kfc.c new file mode 100644 index 000000000..80d202679 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/shared/mem-alloc/ems/ems_kfc.c @@ -0,0 +1,297 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "ems_gc_internal.h" + +static gc_handle_t +gc_init_internal(gc_heap_t *heap, char *base_addr, gc_size_t heap_max_size) +{ + hmu_tree_node_t *root = NULL, *q = NULL; + int ret; + + memset(heap, 0, sizeof *heap); + + ret = os_mutex_init(&heap->lock); + if (ret != BHT_OK) { + os_printf("[GC_ERROR]failed to init lock\n"); + return NULL; + } + + /* init all data structures*/ + heap->current_size = heap_max_size; + heap->base_addr = (gc_uint8 *)base_addr; + heap->heap_id = (gc_handle_t)heap; + + heap->total_free_size = heap->current_size; + heap->highmark_size = 0; + + root = heap->kfc_tree_root = (hmu_tree_node_t *)heap->kfc_tree_root_buf; + memset(root, 0, sizeof *root); + root->size = sizeof *root; + hmu_set_ut(&root->hmu_header, HMU_FC); + hmu_set_size(&root->hmu_header, sizeof *root); + + q = (hmu_tree_node_t *)heap->base_addr; + memset(q, 0, sizeof *q); + hmu_set_ut(&q->hmu_header, HMU_FC); + hmu_set_size(&q->hmu_header, heap->current_size); + + ASSERT_TREE_NODE_ALIGNED_ACCESS(q); + ASSERT_TREE_NODE_ALIGNED_ACCESS(root); + + hmu_mark_pinuse(&q->hmu_header); + root->right = q; + q->parent = root; + q->size = heap->current_size; + + bh_assert(root->size <= HMU_FC_NORMAL_MAX_SIZE); + + return heap; +} + +gc_handle_t +gc_init_with_pool(char *buf, gc_size_t buf_size) +{ + char *buf_end = buf + buf_size; + char *buf_aligned = (char *)(((uintptr_t)buf + 7) & (uintptr_t)~7); + char *base_addr = buf_aligned + sizeof(gc_heap_t); + gc_heap_t *heap = (gc_heap_t *)buf_aligned; + gc_size_t heap_max_size; + + if (buf_size < APP_HEAP_SIZE_MIN) { + os_printf("[GC_ERROR]heap init buf size (%" PRIu32 ") < %" PRIu32 "\n", + buf_size, (uint32)APP_HEAP_SIZE_MIN); + return NULL; + } + + base_addr = + (char *)(((uintptr_t)base_addr + 7) & (uintptr_t)~7) + GC_HEAD_PADDING; + heap_max_size = (uint32)(buf_end - base_addr) & (uint32)~7; + +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Heap created, total size: %u\n", buf_size); + os_printf(" heap struct size: %u\n", sizeof(gc_heap_t)); + os_printf(" actual heap size: %u\n", heap_max_size); + os_printf(" padding bytes: %u\n", + buf_size - sizeof(gc_heap_t) - heap_max_size); +#endif + return gc_init_internal(heap, base_addr, heap_max_size); +} + +gc_handle_t +gc_init_with_struct_and_pool(char *struct_buf, gc_size_t struct_buf_size, + char *pool_buf, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t *)struct_buf; + char *base_addr = pool_buf + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf + pool_buf_size; + gc_size_t heap_max_size; + + if ((((uintptr_t)struct_buf) & 7) != 0) { + os_printf("[GC_ERROR]heap init struct buf not 8-byte aligned\n"); + return NULL; + } + + if (struct_buf_size < sizeof(gc_handle_t)) { + os_printf("[GC_ERROR]heap init struct buf size (%" PRIu32 ") < %zu\n", + struct_buf_size, sizeof(gc_handle_t)); + return NULL; + } + + if ((((uintptr_t)pool_buf) & 7) != 0) { + os_printf("[GC_ERROR]heap init pool buf not 8-byte aligned\n"); + return NULL; + } + + if (pool_buf_size < APP_HEAP_SIZE_MIN) { + os_printf("[GC_ERROR]heap init buf size (%" PRIu32 ") < %u\n", + pool_buf_size, APP_HEAP_SIZE_MIN); + return NULL; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr) & (uint32)~7; + +#if WASM_ENABLE_MEMORY_TRACING != 0 + os_printf("Heap created, total size: %u\n", + struct_buf_size + pool_buf_size); + os_printf(" heap struct size: %u\n", sizeof(gc_heap_t)); + os_printf(" actual heap size: %u\n", heap_max_size); + os_printf(" padding bytes: %u\n", pool_buf_size - heap_max_size); +#endif + return gc_init_internal(heap, base_addr, heap_max_size); +} + +int +gc_destroy_with_pool(gc_handle_t handle) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + int ret = GC_SUCCESS; + +#if BH_ENABLE_GC_VERIFY != 0 + hmu_t *cur = (hmu_t *)heap->base_addr; + hmu_t *end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + if (!heap->is_heap_corrupted + && (hmu_t *)((char *)cur + hmu_get_size(cur)) != end) { + os_printf("Memory leak detected:\n"); + gci_dump(heap); + ret = GC_ERROR; + } +#endif + + os_mutex_destroy(&heap->lock); + memset(heap, 0, sizeof(gc_heap_t)); + return ret; +} + +uint32 +gc_get_heap_struct_size() +{ + return sizeof(gc_heap_t); +} + +static void +adjust_ptr(uint8 **p_ptr, intptr_t offset) +{ + if (*p_ptr) + *p_ptr = (uint8 *)((intptr_t)(*p_ptr) + offset); +} + +int +gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + char *base_addr_new = pool_buf_new + GC_HEAD_PADDING; + char *pool_buf_end = pool_buf_new + pool_buf_size; + intptr_t offset = (uint8 *)base_addr_new - (uint8 *)heap->base_addr; + hmu_t *cur = NULL, *end = NULL; + hmu_tree_node_t *tree_node; + uint8 **p_left, **p_right, **p_parent; + gc_size_t heap_max_size, size; + + if ((((uintptr_t)pool_buf_new) & 7) != 0) { + os_printf("[GC_ERROR]heap migrate pool buf not 8-byte aligned\n"); + return GC_ERROR; + } + + heap_max_size = (uint32)(pool_buf_end - base_addr_new) & (uint32)~7; + + if (pool_buf_end < base_addr_new || heap_max_size < heap->current_size) { + os_printf("[GC_ERROR]heap migrate invlaid pool buf size\n"); + return GC_ERROR; + } + + if (offset == 0) + return 0; + + if (heap->is_heap_corrupted) { + os_printf("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + return GC_ERROR; + } + + heap->base_addr = (uint8 *)base_addr_new; + + ASSERT_TREE_NODE_ALIGNED_ACCESS(heap->kfc_tree_root); + + p_left = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, left)); + p_right = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, right)); + p_parent = (uint8 **)((uint8 *)heap->kfc_tree_root + + offsetof(hmu_tree_node_t, parent)); + adjust_ptr(p_left, offset); + adjust_ptr(p_right, offset); + adjust_ptr(p_parent, offset); + + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)((char *)heap->base_addr + heap->current_size); + + while (cur < end) { + size = hmu_get_size(cur); + + if (size <= 0 || size > (uint32)((uint8 *)end - (uint8 *)cur)) { + os_printf("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + heap->is_heap_corrupted = true; + return GC_ERROR; + } + + if (hmu_get_ut(cur) == HMU_FC && !HMU_IS_FC_NORMAL(size)) { + tree_node = (hmu_tree_node_t *)cur; + + ASSERT_TREE_NODE_ALIGNED_ACCESS(tree_node); + + p_left = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, left)); + p_right = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, right)); + p_parent = (uint8 **)((uint8 *)tree_node + + offsetof(hmu_tree_node_t, parent)); + adjust_ptr(p_left, offset); + adjust_ptr(p_right, offset); + if (tree_node->parent != heap->kfc_tree_root) + /* The root node belongs to heap structure, + it is fixed part and isn't changed. */ + adjust_ptr(p_parent, offset); + } + cur = (hmu_t *)((char *)cur + size); + } + + if (cur != end) { + os_printf("[GC_ERROR]Heap is corrupted, heap migrate failed.\n"); + heap->is_heap_corrupted = true; + return GC_ERROR; + } + + return 0; +} + +bool +gc_is_heap_corrupted(gc_handle_t handle) +{ + gc_heap_t *heap = (gc_heap_t *)handle; + + return heap->is_heap_corrupted ? true : false; +} + +#if BH_ENABLE_GC_VERIFY != 0 +void +gci_verify_heap(gc_heap_t *heap) +{ + hmu_t *cur = NULL, *end = NULL; + + bh_assert(heap && gci_is_heap_valid(heap)); + cur = (hmu_t *)heap->base_addr; + end = (hmu_t *)(heap->base_addr + heap->current_size); + while (cur < end) { + hmu_verify(heap, cur); + cur = (hmu_t *)((gc_uint8 *)cur + hmu_get_size(cur)); + } + bh_assert(cur == end); +} +#endif + +void * +gc_heap_stats(void *heap_arg, uint32 *stats, int size) +{ + int i; + gc_heap_t *heap = (gc_heap_t *)heap_arg; + + for (i = 0; i < size; i++) { + switch (i) { + case GC_STAT_TOTAL: + stats[i] = heap->current_size; + break; + case GC_STAT_FREE: + stats[i] = heap->total_free_size; + break; + case GC_STAT_HIGHMARK: + stats[i] = heap->highmark_size; + break; + default: + break; + } + } + return heap; +} |