summaryrefslogtreecommitdiffstats
path: root/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_llvm.c
diff options
context:
space:
mode:
Diffstat (limited to 'fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_llvm.c')
-rw-r--r--fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_llvm.c2770
1 files changed, 2770 insertions, 0 deletions
diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_llvm.c b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_llvm.c
new file mode 100644
index 000000000..dc3fe7f59
--- /dev/null
+++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_llvm.c
@@ -0,0 +1,2770 @@
+/*
+ * Copyright (C) 2019 Intel Corporation. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ */
+
+#include "aot_llvm.h"
+#include "aot_llvm_extra2.h"
+#include "aot_compiler.h"
+#include "aot_emit_exception.h"
+#include "../aot/aot_runtime.h"
+#include "../aot/aot_intrinsic.h"
+
+#if WASM_ENABLE_DEBUG_AOT != 0
+#include "debug/dwarf_extractor.h"
+#endif
+
+LLVMTypeRef
+wasm_type_to_llvm_type(AOTLLVMTypes *llvm_types, uint8 wasm_type)
+{
+ switch (wasm_type) {
+ case VALUE_TYPE_I32:
+ case VALUE_TYPE_FUNCREF:
+ case VALUE_TYPE_EXTERNREF:
+ return llvm_types->int32_type;
+ case VALUE_TYPE_I64:
+ return llvm_types->int64_type;
+ case VALUE_TYPE_F32:
+ return llvm_types->float32_type;
+ case VALUE_TYPE_F64:
+ return llvm_types->float64_type;
+ case VALUE_TYPE_V128:
+ return llvm_types->i64x2_vec_type;
+ case VALUE_TYPE_VOID:
+ return llvm_types->void_type;
+ default:
+ break;
+ }
+ return NULL;
+}
+
+/**
+ * Add LLVM function
+ */
+static LLVMValueRef
+aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module,
+ AOTFuncType *aot_func_type, uint32 func_index,
+ LLVMTypeRef *p_func_type)
+{
+ LLVMValueRef func = NULL;
+ LLVMTypeRef *param_types, ret_type, func_type;
+ LLVMValueRef local_value;
+ LLVMTypeRef func_type_wrapper;
+ LLVMValueRef func_wrapper;
+ LLVMBasicBlockRef func_begin;
+ char func_name[48];
+ uint64 size;
+ uint32 i, j = 0, param_count = (uint64)aot_func_type->param_count;
+ uint32 backend_thread_num, compile_thread_num;
+
+ /* exec env as first parameter */
+ param_count++;
+
+ /* Extra wasm function results(except the first one)'s address are
+ * appended to aot function parameters. */
+ if (aot_func_type->result_count > 1)
+ param_count += aot_func_type->result_count - 1;
+
+ /* Initialize parameter types of the LLVM function */
+ size = sizeof(LLVMTypeRef) * ((uint64)param_count);
+ if (size >= UINT32_MAX
+ || !(param_types = wasm_runtime_malloc((uint32)size))) {
+ aot_set_last_error("allocate memory failed.");
+ return NULL;
+ }
+
+ /* exec env as first parameter */
+ param_types[j++] = comp_ctx->exec_env_type;
+ for (i = 0; i < aot_func_type->param_count; i++)
+ param_types[j++] = TO_LLVM_TYPE(aot_func_type->types[i]);
+ /* Extra results' address */
+ for (i = 1; i < aot_func_type->result_count; i++, j++) {
+ param_types[j] =
+ TO_LLVM_TYPE(aot_func_type->types[aot_func_type->param_count + i]);
+ if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) {
+ aot_set_last_error("llvm get pointer type failed.");
+ goto fail;
+ }
+ }
+
+ /* Resolve return type of the LLVM function */
+ if (aot_func_type->result_count)
+ ret_type =
+ TO_LLVM_TYPE(aot_func_type->types[aot_func_type->param_count]);
+ else
+ ret_type = VOID_TYPE;
+
+ /* Resolve function prototype */
+ if (!(func_type =
+ LLVMFunctionType(ret_type, param_types, param_count, false))) {
+ aot_set_last_error("create LLVM function type failed.");
+ goto fail;
+ }
+
+ /* Add LLVM function */
+ snprintf(func_name, sizeof(func_name), "%s%d", AOT_FUNC_PREFIX, func_index);
+ if (!(func = LLVMAddFunction(module, func_name, func_type))) {
+ aot_set_last_error("add LLVM function failed.");
+ goto fail;
+ }
+
+ j = 0;
+ local_value = LLVMGetParam(func, j++);
+ LLVMSetValueName(local_value, "exec_env");
+
+ /* Set parameter names */
+ for (i = 0; i < aot_func_type->param_count; i++) {
+ local_value = LLVMGetParam(func, j++);
+ LLVMSetValueName(local_value, "");
+ }
+
+ if (p_func_type)
+ *p_func_type = func_type;
+
+ backend_thread_num = WASM_ORC_JIT_BACKEND_THREAD_NUM;
+ compile_thread_num = WASM_ORC_JIT_COMPILE_THREAD_NUM;
+
+ /* Add the jit wrapper function with simple prototype, so that we
+ can easily call it to trigger its compilation and let LLVM JIT
+ compile the actual jit functions by adding them into the function
+ list in the PartitionFunction callback */
+ if (comp_ctx->is_jit_mode
+ && (func_index % (backend_thread_num * compile_thread_num)
+ < backend_thread_num)) {
+ func_type_wrapper = LLVMFunctionType(VOID_TYPE, NULL, 0, false);
+ if (!func_type_wrapper) {
+ aot_set_last_error("create LLVM function type failed.");
+ goto fail;
+ }
+
+ snprintf(func_name, sizeof(func_name), "%s%d%s", AOT_FUNC_PREFIX,
+ func_index, "_wrapper");
+ if (!(func_wrapper =
+ LLVMAddFunction(module, func_name, func_type_wrapper))) {
+ aot_set_last_error("add LLVM function failed.");
+ goto fail;
+ }
+
+ if (!(func_begin = LLVMAppendBasicBlockInContext(
+ comp_ctx->context, func_wrapper, "func_begin"))) {
+ aot_set_last_error("add LLVM basic block failed.");
+ goto fail;
+ }
+
+ LLVMPositionBuilderAtEnd(comp_ctx->builder, func_begin);
+ if (!LLVMBuildRetVoid(comp_ctx->builder)) {
+ aot_set_last_error("llvm build ret failed.");
+ goto fail;
+ }
+ }
+
+fail:
+ wasm_runtime_free(param_types);
+ return func;
+}
+
+static void
+free_block_memory(AOTBlock *block)
+{
+ if (block->param_types)
+ wasm_runtime_free(block->param_types);
+ if (block->result_types)
+ wasm_runtime_free(block->result_types);
+ wasm_runtime_free(block);
+}
+
+/**
+ * Create first AOTBlock, or function block for the function
+ */
+static AOTBlock *
+aot_create_func_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
+ AOTFunc *func, AOTFuncType *aot_func_type)
+{
+ AOTBlock *aot_block;
+ uint32 param_count = aot_func_type->param_count,
+ result_count = aot_func_type->result_count;
+
+ /* Allocate memory */
+ if (!(aot_block = wasm_runtime_malloc(sizeof(AOTBlock)))) {
+ aot_set_last_error("allocate memory failed.");
+ return NULL;
+ }
+ memset(aot_block, 0, sizeof(AOTBlock));
+ if (param_count
+ && !(aot_block->param_types = wasm_runtime_malloc(param_count))) {
+ aot_set_last_error("allocate memory failed.");
+ goto fail;
+ }
+ if (result_count) {
+ if (!(aot_block->result_types = wasm_runtime_malloc(result_count))) {
+ aot_set_last_error("allocate memory failed.");
+ goto fail;
+ }
+ }
+
+ /* Set block data */
+ aot_block->label_type = LABEL_TYPE_FUNCTION;
+ aot_block->param_count = param_count;
+ if (param_count) {
+ bh_memcpy_s(aot_block->param_types, param_count, aot_func_type->types,
+ param_count);
+ }
+ aot_block->result_count = result_count;
+ if (result_count) {
+ bh_memcpy_s(aot_block->result_types, result_count,
+ aot_func_type->types + param_count, result_count);
+ }
+ aot_block->wasm_code_end = func->code + func->code_size;
+
+ /* Add function entry block */
+ if (!(aot_block->llvm_entry_block = LLVMAppendBasicBlockInContext(
+ comp_ctx->context, func_ctx->func, "func_begin"))) {
+ aot_set_last_error("add LLVM basic block failed.");
+ goto fail;
+ }
+
+ return aot_block;
+
+fail:
+ free_block_memory(aot_block);
+ return NULL;
+}
+
+static bool
+create_argv_buf(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef argv_buf_offset = I32_THREE, argv_buf_addr;
+ LLVMTypeRef int32_ptr_type;
+
+ /* Get argv buffer address */
+ if (!(argv_buf_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env,
+ &argv_buf_offset, 1, "argv_buf_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+
+ if (!(int32_ptr_type = LLVMPointerType(INT32_PTR_TYPE, 0))) {
+ aot_set_last_error("llvm add pointer type failed");
+ return false;
+ }
+
+ /* Convert to int32 pointer type */
+ if (!(argv_buf_addr = LLVMBuildBitCast(comp_ctx->builder, argv_buf_addr,
+ int32_ptr_type, "argv_buf_ptr"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+
+ if (!(func_ctx->argv_buf = LLVMBuildLoad2(comp_ctx->builder, INT32_PTR_TYPE,
+ argv_buf_addr, "argv_buf"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+create_native_stack_bound(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef stack_bound_offset = I32_FOUR, stack_bound_addr;
+
+ if (!(stack_bound_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env,
+ &stack_bound_offset, 1, "stack_bound_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+
+ if (!(func_ctx->native_stack_bound =
+ LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, stack_bound_addr,
+ "native_stack_bound"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+create_native_stack_top_min(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef offset = I32_NINE;
+
+ if (!(func_ctx->native_stack_top_min_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1,
+ "native_stack_top_min_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+create_aux_stack_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef aux_stack_bound_offset = I32_SIX, aux_stack_bound_addr;
+ LLVMValueRef aux_stack_bottom_offset = I32_SEVEN, aux_stack_bottom_addr;
+
+ /* Get aux stack boundary address */
+ if (!(aux_stack_bound_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env,
+ &aux_stack_bound_offset, 1, "aux_stack_bound_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+
+ if (!(aux_stack_bound_addr =
+ LLVMBuildBitCast(comp_ctx->builder, aux_stack_bound_addr,
+ INT32_PTR_TYPE, "aux_stack_bound_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+
+ if (!(func_ctx->aux_stack_bound =
+ LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, aux_stack_bound_addr,
+ "aux_stack_bound"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+
+ /* Get aux stack bottom address */
+ if (!(aux_stack_bottom_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env,
+ &aux_stack_bottom_offset, 1, "aux_stack_bottom_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+
+ if (!(aux_stack_bottom_addr =
+ LLVMBuildBitCast(comp_ctx->builder, aux_stack_bottom_addr,
+ INT32_PTR_TYPE, "aux_stack_bottom_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (!(func_ctx->aux_stack_bottom =
+ LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, aux_stack_bottom_addr,
+ "aux_stack_bottom"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+create_native_symbol(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef native_symbol_offset = I32_EIGHT, native_symbol_addr;
+
+ if (!(native_symbol_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env,
+ &native_symbol_offset, 1, "native_symbol_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+
+ if (!(func_ctx->native_symbol =
+ LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
+ native_symbol_addr, "native_symbol_tmp"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+
+ if (!(func_ctx->native_symbol =
+ LLVMBuildBitCast(comp_ctx->builder, func_ctx->native_symbol,
+ comp_ctx->exec_env_type, "native_symbol"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+create_local_variables(AOTCompData *comp_data, AOTCompContext *comp_ctx,
+ AOTFuncContext *func_ctx, AOTFunc *func)
+{
+ AOTFuncType *aot_func_type = comp_data->func_types[func->func_type_index];
+ char local_name[32];
+ uint32 i, j = 1;
+
+ for (i = 0; i < aot_func_type->param_count; i++, j++) {
+ snprintf(local_name, sizeof(local_name), "l%d", i);
+ func_ctx->locals[i] =
+ LLVMBuildAlloca(comp_ctx->builder,
+ TO_LLVM_TYPE(aot_func_type->types[i]), local_name);
+ if (!func_ctx->locals[i]) {
+ aot_set_last_error("llvm build alloca failed.");
+ return false;
+ }
+ if (!LLVMBuildStore(comp_ctx->builder, LLVMGetParam(func_ctx->func, j),
+ func_ctx->locals[i])) {
+ aot_set_last_error("llvm build store failed.");
+ return false;
+ }
+ }
+
+ for (i = 0; i < func->local_count; i++) {
+ LLVMTypeRef local_type;
+ LLVMValueRef local_value = NULL;
+ snprintf(local_name, sizeof(local_name), "l%d",
+ aot_func_type->param_count + i);
+ local_type = TO_LLVM_TYPE(func->local_types[i]);
+ func_ctx->locals[aot_func_type->param_count + i] =
+ LLVMBuildAlloca(comp_ctx->builder, local_type, local_name);
+ if (!func_ctx->locals[aot_func_type->param_count + i]) {
+ aot_set_last_error("llvm build alloca failed.");
+ return false;
+ }
+ switch (func->local_types[i]) {
+ case VALUE_TYPE_I32:
+ local_value = I32_ZERO;
+ break;
+ case VALUE_TYPE_I64:
+ local_value = I64_ZERO;
+ break;
+ case VALUE_TYPE_F32:
+ local_value = F32_ZERO;
+ break;
+ case VALUE_TYPE_F64:
+ local_value = F64_ZERO;
+ break;
+ case VALUE_TYPE_V128:
+ local_value = V128_i64x2_ZERO;
+ break;
+ case VALUE_TYPE_FUNCREF:
+ case VALUE_TYPE_EXTERNREF:
+ local_value = REF_NULL;
+ break;
+ default:
+ bh_assert(0);
+ break;
+ }
+ if (!LLVMBuildStore(comp_ctx->builder, local_value,
+ func_ctx->locals[aot_func_type->param_count + i])) {
+ aot_set_last_error("llvm build store failed.");
+ return false;
+ }
+ }
+
+ if (comp_ctx->enable_stack_bound_check
+ || comp_ctx->enable_stack_estimation) {
+ if (aot_func_type->param_count + func->local_count > 0) {
+ func_ctx->last_alloca = func_ctx->locals[aot_func_type->param_count
+ + func->local_count - 1];
+ if (!(func_ctx->last_alloca =
+ LLVMBuildBitCast(comp_ctx->builder, func_ctx->last_alloca,
+ INT8_PTR_TYPE, "stack_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed.");
+ return false;
+ }
+ }
+ else {
+ if (!(func_ctx->last_alloca = LLVMBuildAlloca(
+ comp_ctx->builder, INT8_TYPE, "stack_ptr"))) {
+ aot_set_last_error("llvm build alloca failed.");
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+static bool
+create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
+ LLVMTypeRef int8_ptr_type, uint32 func_index)
+{
+ LLVMValueRef offset, mem_info_base;
+ uint32 memory_count;
+ WASMModule *module = comp_ctx->comp_data->wasm_module;
+ WASMFunction *func = module->functions[func_index];
+ LLVMTypeRef bound_check_type;
+ bool mem_space_unchanged =
+ (!func->has_op_memory_grow && !func->has_op_func_call)
+ || (!module->possible_memory_grow);
+#if WASM_ENABLE_SHARED_MEMORY != 0
+ bool is_shared_memory;
+#endif
+
+ func_ctx->mem_space_unchanged = mem_space_unchanged;
+
+ memory_count = module->memory_count + module->import_memory_count;
+ /* If the module dosen't have memory, reserve
+ one mem_info space with empty content */
+ if (memory_count == 0)
+ memory_count = 1;
+
+ if (!(func_ctx->mem_info =
+ wasm_runtime_malloc(sizeof(AOTMemInfo) * memory_count))) {
+ return false;
+ }
+ memset(func_ctx->mem_info, 0, sizeof(AOTMemInfo));
+
+ /* Currently we only create memory info for memory 0 */
+ /* Load memory base address */
+#if WASM_ENABLE_SHARED_MEMORY != 0
+ is_shared_memory =
+ comp_ctx->comp_data->memories[0].memory_flags & 0x02 ? true : false;
+ if (is_shared_memory) {
+ LLVMValueRef shared_mem_addr;
+ offset = I32_CONST(offsetof(AOTModuleInstance, memories));
+ if (!offset) {
+ aot_set_last_error("create llvm const failed.");
+ return false;
+ }
+
+ /* aot_inst->memories */
+ if (!(shared_mem_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1,
+ "shared_mem_addr_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ if (!(shared_mem_addr =
+ LLVMBuildBitCast(comp_ctx->builder, shared_mem_addr,
+ int8_ptr_type, "shared_mem_addr_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ /* aot_inst->memories[0] */
+ if (!(shared_mem_addr =
+ LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
+ shared_mem_addr, "shared_mem_addr"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ if (!(shared_mem_addr =
+ LLVMBuildBitCast(comp_ctx->builder, shared_mem_addr,
+ int8_ptr_type, "shared_mem_addr_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (!(shared_mem_addr =
+ LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
+ shared_mem_addr, "shared_mem_addr"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ /* memories[0]->memory_data */
+ offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1,
+ "mem_base_addr_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ /* memories[0]->cur_page_count */
+ offset = I32_CONST(offsetof(AOTMemoryInstance, cur_page_count));
+ if (!(func_ctx->mem_info[0].mem_cur_page_count_addr =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
+ shared_mem_addr, &offset, 1,
+ "mem_cur_page_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ /* memories[0]->memory_data_size */
+ offset = I32_CONST(offsetof(AOTMemoryInstance, memory_data_size));
+ if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, INT8_TYPE, shared_mem_addr, &offset, 1,
+ "mem_data_size_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ }
+ else
+#endif
+ {
+ uint32 offset_of_global_table_data;
+
+ if (comp_ctx->is_jit_mode)
+ offset_of_global_table_data =
+ offsetof(WASMModuleInstance, global_table_data);
+ else
+ offset_of_global_table_data =
+ offsetof(AOTModuleInstance, global_table_data);
+
+ offset = I32_CONST(offset_of_global_table_data
+ + offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1,
+ "mem_base_addr_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ offset = I32_CONST(offset_of_global_table_data
+ + offsetof(AOTMemoryInstance, cur_page_count));
+ if (!(func_ctx->mem_info[0].mem_cur_page_count_addr =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
+ func_ctx->aot_inst, &offset, 1,
+ "mem_cur_page_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ offset = I32_CONST(offset_of_global_table_data
+ + offsetof(AOTMemoryInstance, memory_data_size));
+ if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1,
+ "mem_data_size_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ }
+ /* Store mem info base address before cast */
+ mem_info_base = func_ctx->mem_info[0].mem_base_addr;
+
+ if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_base_addr,
+ int8_ptr_type, "mem_base_addr_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_cur_page_count_addr = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_cur_page_count_addr,
+ INT32_PTR_TYPE, "mem_cur_page_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_data_size_addr,
+ INT32_PTR_TYPE, "mem_data_size_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (mem_space_unchanged) {
+ if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildLoad2(
+ comp_ctx->builder, OPQ_PTR_TYPE,
+ func_ctx->mem_info[0].mem_base_addr, "mem_base_addr"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_cur_page_count_addr =
+ LLVMBuildLoad2(comp_ctx->builder, I32_TYPE,
+ func_ctx->mem_info[0].mem_cur_page_count_addr,
+ "mem_cur_page_count"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_data_size_addr = LLVMBuildLoad2(
+ comp_ctx->builder, I32_TYPE,
+ func_ctx->mem_info[0].mem_data_size_addr, "mem_data_size"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+#if WASM_ENABLE_SHARED_MEMORY != 0
+ else if (is_shared_memory) {
+ /* The base address for shared memory will never changed,
+ we can load the value here */
+ if (!(func_ctx->mem_info[0].mem_base_addr = LLVMBuildLoad2(
+ comp_ctx->builder, OPQ_PTR_TYPE,
+ func_ctx->mem_info[0].mem_base_addr, "mem_base_addr"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+#endif
+
+ bound_check_type = (comp_ctx->pointer_size == sizeof(uint64))
+ ? INT64_PTR_TYPE
+ : INT32_PTR_TYPE;
+
+ /* Load memory bound check constants */
+ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_1byte)
+ - offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_bound_check_1byte =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
+ &offset, 1, "bound_check_1byte_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_1byte,
+ bound_check_type, "bound_check_1byte_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (mem_space_unchanged) {
+ if (!(func_ctx->mem_info[0].mem_bound_check_1byte = LLVMBuildLoad2(
+ comp_ctx->builder,
+ (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE
+ : I32_TYPE,
+ func_ctx->mem_info[0].mem_bound_check_1byte,
+ "bound_check_1byte"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+
+ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_2bytes)
+ - offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_bound_check_2bytes =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
+ &offset, 1, "bound_check_2bytes_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_2bytes,
+ bound_check_type, "bound_check_2bytes_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (mem_space_unchanged) {
+ if (!(func_ctx->mem_info[0].mem_bound_check_2bytes = LLVMBuildLoad2(
+ comp_ctx->builder,
+ (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE
+ : I32_TYPE,
+ func_ctx->mem_info[0].mem_bound_check_2bytes,
+ "bound_check_2bytes"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+
+ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_4bytes)
+ - offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_bound_check_4bytes =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
+ &offset, 1, "bound_check_4bytes_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_4bytes,
+ bound_check_type, "bound_check_4bytes_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (mem_space_unchanged) {
+ if (!(func_ctx->mem_info[0].mem_bound_check_4bytes = LLVMBuildLoad2(
+ comp_ctx->builder,
+ (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE
+ : I32_TYPE,
+ func_ctx->mem_info[0].mem_bound_check_4bytes,
+ "bound_check_4bytes"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+
+ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_8bytes)
+ - offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_bound_check_8bytes =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, mem_info_base,
+ &offset, 1, "bound_check_8bytes_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_8bytes,
+ bound_check_type, "bound_check_8bytes_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (mem_space_unchanged) {
+ if (!(func_ctx->mem_info[0].mem_bound_check_8bytes = LLVMBuildLoad2(
+ comp_ctx->builder,
+ (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE
+ : I32_TYPE,
+ func_ctx->mem_info[0].mem_bound_check_8bytes,
+ "bound_check_8bytes"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+
+ offset = I32_CONST(offsetof(AOTMemoryInstance, mem_bound_check_16bytes)
+ - offsetof(AOTMemoryInstance, memory_data));
+ if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, INT8_TYPE, mem_info_base, &offset, 1,
+ "bound_check_16bytes_offset"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ return false;
+ }
+ if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildBitCast(
+ comp_ctx->builder, func_ctx->mem_info[0].mem_bound_check_16bytes,
+ bound_check_type, "bound_check_16bytes_ptr"))) {
+ aot_set_last_error("llvm build bit cast failed");
+ return false;
+ }
+ if (mem_space_unchanged) {
+ if (!(func_ctx->mem_info[0].mem_bound_check_16bytes = LLVMBuildLoad2(
+ comp_ctx->builder,
+ (comp_ctx->pointer_size == sizeof(uint64)) ? I64_TYPE
+ : I32_TYPE,
+ func_ctx->mem_info[0].mem_bound_check_16bytes,
+ "bound_check_16bytes"))) {
+ aot_set_last_error("llvm build load failed");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool
+create_cur_exception(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef offset;
+
+ offset = I32_CONST(offsetof(AOTModuleInstance, cur_exception));
+ func_ctx->cur_exception =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst,
+ &offset, 1, "cur_exception");
+ if (!func_ctx->cur_exception) {
+ aot_set_last_error("llvm build in bounds gep failed.");
+ return false;
+ }
+ return true;
+}
+
+static bool
+create_func_type_indexes(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef offset, func_type_indexes_ptr;
+ LLVMTypeRef int32_ptr_type;
+
+ offset = I32_CONST(offsetof(AOTModuleInstance, func_type_indexes));
+ func_type_indexes_ptr =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst,
+ &offset, 1, "func_type_indexes_ptr");
+ if (!func_type_indexes_ptr) {
+ aot_set_last_error("llvm build add failed.");
+ return false;
+ }
+
+ if (!(int32_ptr_type = LLVMPointerType(INT32_PTR_TYPE, 0))) {
+ aot_set_last_error("llvm get pointer type failed.");
+ return false;
+ }
+
+ func_ctx->func_type_indexes =
+ LLVMBuildBitCast(comp_ctx->builder, func_type_indexes_ptr,
+ int32_ptr_type, "func_type_indexes_tmp");
+ if (!func_ctx->func_type_indexes) {
+ aot_set_last_error("llvm build bit cast failed.");
+ return false;
+ }
+
+ func_ctx->func_type_indexes =
+ LLVMBuildLoad2(comp_ctx->builder, INT32_PTR_TYPE,
+ func_ctx->func_type_indexes, "func_type_indexes");
+ if (!func_ctx->func_type_indexes) {
+ aot_set_last_error("llvm build load failed.");
+ return false;
+ }
+ return true;
+}
+
+static bool
+create_func_ptrs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
+{
+ LLVMValueRef offset;
+
+ offset = I32_CONST(offsetof(AOTModuleInstance, func_ptrs));
+ func_ctx->func_ptrs =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst,
+ &offset, 1, "func_ptrs_offset");
+ if (!func_ctx->func_ptrs) {
+ aot_set_last_error("llvm build in bounds gep failed.");
+ return false;
+ }
+ func_ctx->func_ptrs =
+ LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs,
+ comp_ctx->exec_env_type, "func_ptrs_tmp");
+ if (!func_ctx->func_ptrs) {
+ aot_set_last_error("llvm build bit cast failed.");
+ return false;
+ }
+
+ func_ctx->func_ptrs = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
+ func_ctx->func_ptrs, "func_ptrs_ptr");
+ if (!func_ctx->func_ptrs) {
+ aot_set_last_error("llvm build load failed.");
+ return false;
+ }
+
+ func_ctx->func_ptrs =
+ LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs,
+ comp_ctx->exec_env_type, "func_ptrs");
+ if (!func_ctx->func_ptrs) {
+ aot_set_last_error("llvm build bit cast failed.");
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * Create function compiler context
+ */
+static AOTFuncContext *
+aot_create_func_context(AOTCompData *comp_data, AOTCompContext *comp_ctx,
+ AOTFunc *func, uint32 func_index)
+{
+ AOTFuncContext *func_ctx;
+ AOTFuncType *aot_func_type = comp_data->func_types[func->func_type_index];
+ WASMModule *module = comp_ctx->comp_data->wasm_module;
+ WASMFunction *wasm_func = module->functions[func_index];
+ AOTBlock *aot_block;
+ LLVMTypeRef int8_ptr_type;
+ LLVMValueRef aot_inst_offset = I32_TWO, aot_inst_addr;
+ uint64 size;
+
+ /* Allocate memory for the function context */
+ size = offsetof(AOTFuncContext, locals)
+ + sizeof(LLVMValueRef)
+ * ((uint64)aot_func_type->param_count + func->local_count);
+ if (size >= UINT32_MAX || !(func_ctx = wasm_runtime_malloc((uint32)size))) {
+ aot_set_last_error("allocate memory failed.");
+ return NULL;
+ }
+
+ memset(func_ctx, 0, (uint32)size);
+ func_ctx->aot_func = func;
+
+ func_ctx->module = comp_ctx->module;
+
+ /* Add LLVM function */
+ if (!(func_ctx->func =
+ aot_add_llvm_func(comp_ctx, func_ctx->module, aot_func_type,
+ func_index, &func_ctx->func_type))) {
+ goto fail;
+ }
+
+ /* Create function's first AOTBlock */
+ if (!(aot_block =
+ aot_create_func_block(comp_ctx, func_ctx, func, aot_func_type))) {
+ goto fail;
+ }
+
+#if WASM_ENABLE_DEBUG_AOT != 0
+ func_ctx->debug_func = dwarf_gen_func_info(comp_ctx, func_ctx);
+#endif
+
+ aot_block_stack_push(&func_ctx->block_stack, aot_block);
+
+ /* Add local variables */
+ LLVMPositionBuilderAtEnd(comp_ctx->builder, aot_block->llvm_entry_block);
+
+ /* Save the pameters for fast access */
+ func_ctx->exec_env = LLVMGetParam(func_ctx->func, 0);
+
+ /* Get aot inst address, the layout of exec_env is:
+ exec_env->next, exec_env->prev, exec_env->module_inst, and argv_buf */
+ if (!(aot_inst_addr = LLVMBuildInBoundsGEP2(
+ comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env,
+ &aot_inst_offset, 1, "aot_inst_addr"))) {
+ aot_set_last_error("llvm build in bounds gep failed");
+ goto fail;
+ }
+
+ /* Load aot inst */
+ if (!(func_ctx->aot_inst = LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE,
+ aot_inst_addr, "aot_inst"))) {
+ aot_set_last_error("llvm build load failed");
+ goto fail;
+ }
+
+ /* Get argv buffer address */
+ if (wasm_func->has_op_func_call && !create_argv_buf(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ /* Get native stack boundary address */
+ if (comp_ctx->enable_stack_bound_check
+ && !create_native_stack_bound(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+ if (comp_ctx->enable_stack_estimation
+ && !create_native_stack_top_min(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ /* Get auxiliary stack info */
+ if (wasm_func->has_op_set_global_aux_stack
+ && !create_aux_stack_info(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ /* Get native symbol list */
+ if (comp_ctx->is_indirect_mode
+ && !create_native_symbol(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ /* Create local variables */
+ if (!create_local_variables(comp_data, comp_ctx, func_ctx, func)) {
+ goto fail;
+ }
+
+ if (!(int8_ptr_type = LLVMPointerType(INT8_PTR_TYPE, 0))) {
+ aot_set_last_error("llvm add pointer type failed.");
+ goto fail;
+ }
+
+ /* Create base addr, end addr, data size of mem, heap */
+ if (wasm_func->has_memory_operations
+ && !create_memory_info(comp_ctx, func_ctx, int8_ptr_type, func_index)) {
+ goto fail;
+ }
+
+ /* Load current exception */
+ if (!create_cur_exception(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ /* Load function type indexes */
+ if (wasm_func->has_op_call_indirect
+ && !create_func_type_indexes(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ /* Load function pointers */
+ if (!create_func_ptrs(comp_ctx, func_ctx)) {
+ goto fail;
+ }
+
+ return func_ctx;
+
+fail:
+ if (func_ctx->mem_info)
+ wasm_runtime_free(func_ctx->mem_info);
+ aot_block_stack_destroy(&func_ctx->block_stack);
+ wasm_runtime_free(func_ctx);
+ return NULL;
+}
+
+static void
+aot_destroy_func_contexts(AOTFuncContext **func_ctxes, uint32 count)
+{
+ uint32 i;
+
+ for (i = 0; i < count; i++)
+ if (func_ctxes[i]) {
+ if (func_ctxes[i]->mem_info)
+ wasm_runtime_free(func_ctxes[i]->mem_info);
+ aot_block_stack_destroy(&func_ctxes[i]->block_stack);
+ aot_checked_addr_list_destroy(func_ctxes[i]);
+ wasm_runtime_free(func_ctxes[i]);
+ }
+ wasm_runtime_free(func_ctxes);
+}
+
+/**
+ * Create function compiler contexts
+ */
+static AOTFuncContext **
+aot_create_func_contexts(AOTCompData *comp_data, AOTCompContext *comp_ctx)
+{
+ AOTFuncContext **func_ctxes;
+ uint64 size;
+ uint32 i;
+
+ /* Allocate memory */
+ size = sizeof(AOTFuncContext *) * (uint64)comp_data->func_count;
+ if (size >= UINT32_MAX
+ || !(func_ctxes = wasm_runtime_malloc((uint32)size))) {
+ aot_set_last_error("allocate memory failed.");
+ return NULL;
+ }
+
+ memset(func_ctxes, 0, size);
+
+ /* Create each function context */
+ for (i = 0; i < comp_data->func_count; i++) {
+ AOTFunc *func = comp_data->funcs[i];
+ if (!(func_ctxes[i] =
+ aot_create_func_context(comp_data, comp_ctx, func, i))) {
+ aot_destroy_func_contexts(func_ctxes, comp_data->func_count);
+ return NULL;
+ }
+ }
+
+ return func_ctxes;
+}
+
+static bool
+aot_set_llvm_basic_types(AOTLLVMTypes *basic_types, LLVMContextRef context)
+{
+ basic_types->int1_type = LLVMInt1TypeInContext(context);
+ basic_types->int8_type = LLVMInt8TypeInContext(context);
+ basic_types->int16_type = LLVMInt16TypeInContext(context);
+ basic_types->int32_type = LLVMInt32TypeInContext(context);
+ basic_types->int64_type = LLVMInt64TypeInContext(context);
+ basic_types->float32_type = LLVMFloatTypeInContext(context);
+ basic_types->float64_type = LLVMDoubleTypeInContext(context);
+ basic_types->void_type = LLVMVoidTypeInContext(context);
+
+ basic_types->meta_data_type = LLVMMetadataTypeInContext(context);
+
+ basic_types->int8_ptr_type = LLVMPointerType(basic_types->int8_type, 0);
+
+ if (basic_types->int8_ptr_type) {
+ basic_types->int8_pptr_type =
+ LLVMPointerType(basic_types->int8_ptr_type, 0);
+ }
+
+ basic_types->int16_ptr_type = LLVMPointerType(basic_types->int16_type, 0);
+ basic_types->int32_ptr_type = LLVMPointerType(basic_types->int32_type, 0);
+ basic_types->int64_ptr_type = LLVMPointerType(basic_types->int64_type, 0);
+ basic_types->float32_ptr_type =
+ LLVMPointerType(basic_types->float32_type, 0);
+ basic_types->float64_ptr_type =
+ LLVMPointerType(basic_types->float64_type, 0);
+
+ basic_types->i8x16_vec_type = LLVMVectorType(basic_types->int8_type, 16);
+ basic_types->i16x8_vec_type = LLVMVectorType(basic_types->int16_type, 8);
+ basic_types->i32x4_vec_type = LLVMVectorType(basic_types->int32_type, 4);
+ basic_types->i64x2_vec_type = LLVMVectorType(basic_types->int64_type, 2);
+ basic_types->f32x4_vec_type = LLVMVectorType(basic_types->float32_type, 4);
+ basic_types->f64x2_vec_type = LLVMVectorType(basic_types->float64_type, 2);
+
+ basic_types->v128_type = basic_types->i64x2_vec_type;
+ basic_types->v128_ptr_type = LLVMPointerType(basic_types->v128_type, 0);
+
+ basic_types->i1x2_vec_type = LLVMVectorType(basic_types->int1_type, 2);
+
+ basic_types->funcref_type = LLVMInt32TypeInContext(context);
+ basic_types->externref_type = LLVMInt32TypeInContext(context);
+
+ return (basic_types->int8_ptr_type && basic_types->int8_pptr_type
+ && basic_types->int16_ptr_type && basic_types->int32_ptr_type
+ && basic_types->int64_ptr_type && basic_types->float32_ptr_type
+ && basic_types->float64_ptr_type && basic_types->i8x16_vec_type
+ && basic_types->i16x8_vec_type && basic_types->i32x4_vec_type
+ && basic_types->i64x2_vec_type && basic_types->f32x4_vec_type
+ && basic_types->f64x2_vec_type && basic_types->i1x2_vec_type
+ && basic_types->meta_data_type && basic_types->funcref_type
+ && basic_types->externref_type)
+ ? true
+ : false;
+}
+
+static bool
+aot_create_llvm_consts(AOTLLVMConsts *consts, AOTCompContext *comp_ctx)
+{
+#define CREATE_I1_CONST(name, value) \
+ if (!(consts->i1_##name = \
+ LLVMConstInt(comp_ctx->basic_types.int1_type, value, true))) \
+ return false;
+
+ CREATE_I1_CONST(zero, 0)
+ CREATE_I1_CONST(one, 1)
+#undef CREATE_I1_CONST
+
+ if (!(consts->i8_zero = I8_CONST(0)))
+ return false;
+
+ if (!(consts->f32_zero = F32_CONST(0)))
+ return false;
+
+ if (!(consts->f64_zero = F64_CONST(0)))
+ return false;
+
+#define CREATE_I32_CONST(name, value) \
+ if (!(consts->i32_##name = LLVMConstInt(I32_TYPE, value, true))) \
+ return false;
+
+ CREATE_I32_CONST(min, (uint32)INT32_MIN)
+ CREATE_I32_CONST(neg_one, (uint32)-1)
+ CREATE_I32_CONST(zero, 0)
+ CREATE_I32_CONST(one, 1)
+ CREATE_I32_CONST(two, 2)
+ CREATE_I32_CONST(three, 3)
+ CREATE_I32_CONST(four, 4)
+ CREATE_I32_CONST(five, 5)
+ CREATE_I32_CONST(six, 6)
+ CREATE_I32_CONST(seven, 7)
+ CREATE_I32_CONST(eight, 8)
+ CREATE_I32_CONST(nine, 9)
+ CREATE_I32_CONST(ten, 10)
+ CREATE_I32_CONST(eleven, 11)
+ CREATE_I32_CONST(twelve, 12)
+ CREATE_I32_CONST(thirteen, 13)
+ CREATE_I32_CONST(fourteen, 14)
+ CREATE_I32_CONST(fifteen, 15)
+ CREATE_I32_CONST(31, 31)
+ CREATE_I32_CONST(32, 32)
+#undef CREATE_I32_CONST
+
+#define CREATE_I64_CONST(name, value) \
+ if (!(consts->i64_##name = LLVMConstInt(I64_TYPE, value, true))) \
+ return false;
+
+ CREATE_I64_CONST(min, (uint64)INT64_MIN)
+ CREATE_I64_CONST(neg_one, (uint64)-1)
+ CREATE_I64_CONST(zero, 0)
+ CREATE_I64_CONST(63, 63)
+ CREATE_I64_CONST(64, 64)
+#undef CREATE_I64_CONST
+
+#define CREATE_V128_CONST(name, type) \
+ if (!(consts->name##_vec_zero = LLVMConstNull(type))) \
+ return false; \
+ if (!(consts->name##_undef = LLVMGetUndef(type))) \
+ return false;
+
+ CREATE_V128_CONST(i8x16, V128_i8x16_TYPE)
+ CREATE_V128_CONST(i16x8, V128_i16x8_TYPE)
+ CREATE_V128_CONST(i32x4, V128_i32x4_TYPE)
+ CREATE_V128_CONST(i64x2, V128_i64x2_TYPE)
+ CREATE_V128_CONST(f32x4, V128_f32x4_TYPE)
+ CREATE_V128_CONST(f64x2, V128_f64x2_TYPE)
+#undef CREATE_V128_CONST
+
+#define CREATE_VEC_ZERO_MASK(slot) \
+ { \
+ LLVMTypeRef type = LLVMVectorType(I32_TYPE, slot); \
+ if (!type || !(consts->i32x##slot##_zero = LLVMConstNull(type))) \
+ return false; \
+ }
+
+ CREATE_VEC_ZERO_MASK(16)
+ CREATE_VEC_ZERO_MASK(8)
+ CREATE_VEC_ZERO_MASK(4)
+ CREATE_VEC_ZERO_MASK(2)
+#undef CREATE_VEC_ZERO_MASK
+
+ return true;
+}
+
+typedef struct ArchItem {
+ char *arch;
+ bool support_eb;
+} ArchItem;
+
+/* clang-format off */
+static ArchItem valid_archs[] = {
+ { "x86_64", false },
+ { "i386", false },
+ { "xtensa", false },
+ { "mips", true },
+ { "mipsel", false },
+ { "aarch64v8", false },
+ { "aarch64v8.1", false },
+ { "aarch64v8.2", false },
+ { "aarch64v8.3", false },
+ { "aarch64v8.4", false },
+ { "aarch64v8.5", false },
+ { "aarch64_bev8", false }, /* big endian */
+ { "aarch64_bev8.1", false },
+ { "aarch64_bev8.2", false },
+ { "aarch64_bev8.3", false },
+ { "aarch64_bev8.4", false },
+ { "aarch64_bev8.5", false },
+ { "armv4", true },
+ { "armv4t", true },
+ { "armv5t", true },
+ { "armv5te", true },
+ { "armv5tej", true },
+ { "armv6", true },
+ { "armv6kz", true },
+ { "armv6t2", true },
+ { "armv6k", true },
+ { "armv7", true },
+ { "armv6m", true },
+ { "armv6sm", true },
+ { "armv7em", true },
+ { "armv8a", true },
+ { "armv8r", true },
+ { "armv8m.base", true },
+ { "armv8m.main", true },
+ { "armv8.1m.main", true },
+ { "thumbv4", true },
+ { "thumbv4t", true },
+ { "thumbv5t", true },
+ { "thumbv5te", true },
+ { "thumbv5tej", true },
+ { "thumbv6", true },
+ { "thumbv6kz", true },
+ { "thumbv6t2", true },
+ { "thumbv6k", true },
+ { "thumbv7", true },
+ { "thumbv6m", true },
+ { "thumbv6sm", true },
+ { "thumbv7em", true },
+ { "thumbv8a", true },
+ { "thumbv8r", true },
+ { "thumbv8m.base", true },
+ { "thumbv8m.main", true },
+ { "thumbv8.1m.main", true },
+ { "riscv32", true },
+ { "riscv64", true },
+ { "arc", true }
+};
+
+static const char *valid_abis[] = {
+ "gnu",
+ "eabi",
+ "gnueabihf",
+ "msvc",
+ "ilp32",
+ "ilp32f",
+ "ilp32d",
+ "lp64",
+ "lp64f",
+ "lp64d"
+};
+/* clang-format on */
+
+static void
+print_supported_targets()
+{
+ uint32 i;
+ os_printf("Supported targets:\n");
+ for (i = 0; i < sizeof(valid_archs) / sizeof(ArchItem); i++) {
+ os_printf("%s ", valid_archs[i].arch);
+ if (valid_archs[i].support_eb)
+ os_printf("%seb ", valid_archs[i].arch);
+ }
+ os_printf("\n");
+}
+
+static void
+print_supported_abis()
+{
+ uint32 i;
+ os_printf("Supported ABI: ");
+ for (i = 0; i < sizeof(valid_abis) / sizeof(const char *); i++)
+ os_printf("%s ", valid_abis[i]);
+ os_printf("\n");
+}
+
+static bool
+check_target_arch(const char *target_arch)
+{
+ uint32 i;
+ char *arch;
+ bool support_eb;
+
+ for (i = 0; i < sizeof(valid_archs) / sizeof(ArchItem); i++) {
+ arch = valid_archs[i].arch;
+ support_eb = valid_archs[i].support_eb;
+
+ if (!strncmp(target_arch, arch, strlen(arch))
+ && ((support_eb
+ && (!strcmp(target_arch + strlen(arch), "eb")
+ || !strcmp(target_arch + strlen(arch), "")))
+ || (!support_eb && !strcmp(target_arch + strlen(arch), "")))) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool
+check_target_abi(const char *target_abi)
+{
+ uint32 i;
+ for (i = 0; i < sizeof(valid_abis) / sizeof(char *); i++) {
+ if (!strcmp(target_abi, valid_abis[i]))
+ return true;
+ }
+ return false;
+}
+
+static void
+get_target_arch_from_triple(const char *triple, char *arch_buf, uint32 buf_size)
+{
+ uint32 i = 0;
+ while (*triple != '-' && *triple != '\0' && i < buf_size - 1)
+ arch_buf[i++] = *triple++;
+ /* Make sure buffer is long enough */
+ bh_assert(*triple == '-' || *triple == '\0');
+}
+
+void
+aot_handle_llvm_errmsg(const char *string, LLVMErrorRef err)
+{
+ char *err_msg = LLVMGetErrorMessage(err);
+ aot_set_last_error_v("%s: %s", string, err_msg);
+ LLVMDisposeErrorMessage(err_msg);
+}
+
+static bool
+create_target_machine_detect_host(AOTCompContext *comp_ctx)
+{
+ char *triple = NULL;
+ LLVMTargetRef target = NULL;
+ char *err_msg = NULL;
+ char *cpu = NULL;
+ char *features = NULL;
+ LLVMTargetMachineRef target_machine = NULL;
+ bool ret = false;
+
+ triple = LLVMGetDefaultTargetTriple();
+ if (triple == NULL) {
+ aot_set_last_error("failed to get default target triple.");
+ goto fail;
+ }
+
+ if (LLVMGetTargetFromTriple(triple, &target, &err_msg) != 0) {
+ aot_set_last_error_v("failed to get llvm target from triple %s.",
+ err_msg);
+ LLVMDisposeMessage(err_msg);
+ goto fail;
+ }
+
+ if (!LLVMTargetHasJIT(target)) {
+ aot_set_last_error("unspported JIT on this platform.");
+ goto fail;
+ }
+
+ cpu = LLVMGetHostCPUName();
+ if (cpu == NULL) {
+ aot_set_last_error("failed to get host cpu information.");
+ goto fail;
+ }
+
+ features = LLVMGetHostCPUFeatures();
+ if (features == NULL) {
+ aot_set_last_error("failed to get host cpu features.");
+ goto fail;
+ }
+
+ LOG_VERBOSE("LLVM ORCJIT detected CPU \"%s\", with features \"%s\"\n", cpu,
+ features);
+
+ /* create TargetMachine */
+ target_machine = LLVMCreateTargetMachine(
+ target, triple, cpu, features, LLVMCodeGenLevelDefault,
+ LLVMRelocDefault, LLVMCodeModelJITDefault);
+ if (!target_machine) {
+ aot_set_last_error("failed to create target machine.");
+ goto fail;
+ }
+ comp_ctx->target_machine = target_machine;
+
+ /* Save target arch */
+ get_target_arch_from_triple(triple, comp_ctx->target_arch,
+ sizeof(comp_ctx->target_arch));
+ ret = true;
+
+fail:
+ if (triple)
+ LLVMDisposeMessage(triple);
+ if (features)
+ LLVMDisposeMessage(features);
+ if (cpu)
+ LLVMDisposeMessage(cpu);
+
+ return ret;
+}
+
+static bool
+orc_jit_create(AOTCompContext *comp_ctx)
+{
+ LLVMErrorRef err;
+ LLVMOrcLLLazyJITRef orc_jit = NULL;
+ LLVMOrcLLLazyJITBuilderRef builder = NULL;
+ LLVMOrcJITTargetMachineBuilderRef jtmb = NULL;
+ bool ret = false;
+
+ builder = LLVMOrcCreateLLLazyJITBuilder();
+ if (builder == NULL) {
+ aot_set_last_error("failed to create jit builder.");
+ goto fail;
+ }
+
+ err = LLVMOrcJITTargetMachineBuilderDetectHost(&jtmb);
+ if (err != LLVMErrorSuccess) {
+ aot_handle_llvm_errmsg(
+ "quited to create LLVMOrcJITTargetMachineBuilderRef", err);
+ goto fail;
+ }
+
+ LLVMOrcLLLazyJITBuilderSetNumCompileThreads(
+ builder, WASM_ORC_JIT_COMPILE_THREAD_NUM);
+
+ /* Ownership transfer:
+ LLVMOrcJITTargetMachineBuilderRef -> LLVMOrcLLJITBuilderRef */
+ LLVMOrcLLLazyJITBuilderSetJITTargetMachineBuilder(builder, jtmb);
+ err = LLVMOrcCreateLLLazyJIT(&orc_jit, builder);
+ if (err != LLVMErrorSuccess) {
+ aot_handle_llvm_errmsg("quited to create llvm lazy orcjit instance",
+ err);
+ goto fail;
+ }
+ /* Ownership transfer: LLVMOrcLLJITBuilderRef -> LLVMOrcLLJITRef */
+ builder = NULL;
+
+ /* Ownership transfer: local -> AOTCompContext */
+ comp_ctx->orc_jit = orc_jit;
+ orc_jit = NULL;
+ ret = true;
+
+fail:
+ if (builder)
+ LLVMOrcDisposeLLLazyJITBuilder(builder);
+
+ if (orc_jit)
+ LLVMOrcDisposeLLLazyJIT(orc_jit);
+ return ret;
+}
+
+bool
+aot_compiler_init(void)
+{
+ /* Initialize LLVM environment */
+
+ LLVMInitializeCore(LLVMGetGlobalPassRegistry());
+#if WASM_ENABLE_WAMR_COMPILER != 0
+ /* Init environment of all targets for AOT compiler */
+ LLVMInitializeAllTargetInfos();
+ LLVMInitializeAllTargets();
+ LLVMInitializeAllTargetMCs();
+ LLVMInitializeAllAsmPrinters();
+#else
+ /* Init environment of native for JIT compiler */
+ LLVMInitializeNativeTarget();
+ LLVMInitializeNativeTarget();
+ LLVMInitializeNativeAsmPrinter();
+#endif
+
+ return true;
+}
+
+void
+aot_compiler_destroy(void)
+{
+ LLVMShutdown();
+}
+
+AOTCompContext *
+aot_create_comp_context(AOTCompData *comp_data, aot_comp_option_t option)
+{
+ AOTCompContext *comp_ctx, *ret = NULL;
+ LLVMTargetRef target;
+ char *triple = NULL, *triple_norm, *arch, *abi;
+ char *cpu = NULL, *features, buf[128];
+ char *triple_norm_new = NULL, *cpu_new = NULL;
+ char *err = NULL, *fp_round = "round.tonearest",
+ *fp_exce = "fpexcept.strict";
+ char triple_buf[32] = { 0 }, features_buf[128] = { 0 };
+ uint32 opt_level, size_level, i;
+ LLVMCodeModel code_model;
+ LLVMTargetDataRef target_data_ref;
+
+ /* Allocate memory */
+ if (!(comp_ctx = wasm_runtime_malloc(sizeof(AOTCompContext)))) {
+ aot_set_last_error("allocate memory failed.");
+ return NULL;
+ }
+
+ memset(comp_ctx, 0, sizeof(AOTCompContext));
+ comp_ctx->comp_data = comp_data;
+
+ /* Create LLVM context, module and builder */
+ comp_ctx->orc_thread_safe_context = LLVMOrcCreateNewThreadSafeContext();
+ if (!comp_ctx->orc_thread_safe_context) {
+ aot_set_last_error("create LLVM ThreadSafeContext failed.");
+ goto fail;
+ }
+
+ /* Get a reference to the underlying LLVMContext, note:
+ different from non LAZY JIT mode, no need to dispose this context,
+ if will be disposed when the thread safe context is disposed */
+ if (!(comp_ctx->context = LLVMOrcThreadSafeContextGetContext(
+ comp_ctx->orc_thread_safe_context))) {
+ aot_set_last_error("get context from LLVM ThreadSafeContext failed.");
+ goto fail;
+ }
+
+ if (!(comp_ctx->builder = LLVMCreateBuilderInContext(comp_ctx->context))) {
+ aot_set_last_error("create LLVM builder failed.");
+ goto fail;
+ }
+
+ /* Create LLVM module for each jit function, note:
+ different from non ORC JIT mode, no need to dispose it,
+ it will be disposed when the thread safe context is disposed */
+ if (!(comp_ctx->module = LLVMModuleCreateWithNameInContext(
+ "WASM Module", comp_ctx->context))) {
+ aot_set_last_error("create LLVM module failed.");
+ goto fail;
+ }
+
+ if (BH_LIST_ERROR == bh_list_init(&comp_ctx->native_symbols)) {
+ goto fail;
+ }
+
+#if WASM_ENABLE_DEBUG_AOT != 0
+ if (!(comp_ctx->debug_builder = LLVMCreateDIBuilder(comp_ctx->module))) {
+ aot_set_last_error("create LLVM Debug Infor builder failed.");
+ goto fail;
+ }
+
+ LLVMAddModuleFlag(
+ comp_ctx->module, LLVMModuleFlagBehaviorWarning, "Debug Info Version",
+ strlen("Debug Info Version"),
+ LLVMValueAsMetadata(LLVMConstInt(LLVMInt32Type(), 3, false)));
+
+ comp_ctx->debug_file = dwarf_gen_file_info(comp_ctx);
+ if (!comp_ctx->debug_file) {
+ aot_set_last_error("dwarf generate file info failed");
+ goto fail;
+ }
+ comp_ctx->debug_comp_unit = dwarf_gen_comp_unit_info(comp_ctx);
+ if (!comp_ctx->debug_comp_unit) {
+ aot_set_last_error("dwarf generate compile unit info failed");
+ goto fail;
+ }
+#endif
+
+ if (option->enable_bulk_memory)
+ comp_ctx->enable_bulk_memory = true;
+
+ if (option->enable_thread_mgr)
+ comp_ctx->enable_thread_mgr = true;
+
+ if (option->enable_tail_call)
+ comp_ctx->enable_tail_call = true;
+
+ if (option->enable_ref_types)
+ comp_ctx->enable_ref_types = true;
+
+ if (option->enable_aux_stack_frame)
+ comp_ctx->enable_aux_stack_frame = true;
+
+ if (option->enable_aux_stack_check)
+ comp_ctx->enable_aux_stack_check = true;
+
+ if (option->is_indirect_mode)
+ comp_ctx->is_indirect_mode = true;
+
+ if (option->disable_llvm_intrinsics)
+ comp_ctx->disable_llvm_intrinsics = true;
+
+ if (option->disable_llvm_lto)
+ comp_ctx->disable_llvm_lto = true;
+
+ if (option->enable_stack_estimation)
+ comp_ctx->enable_stack_estimation = true;
+
+ comp_ctx->opt_level = option->opt_level;
+ comp_ctx->size_level = option->size_level;
+
+ comp_ctx->custom_sections_wp = option->custom_sections;
+ comp_ctx->custom_sections_count = option->custom_sections_count;
+
+ if (option->is_jit_mode) {
+ comp_ctx->is_jit_mode = true;
+
+ /* Create TargetMachine */
+ if (!create_target_machine_detect_host(comp_ctx))
+ goto fail;
+
+ /* Create LLJIT Instance */
+ if (!orc_jit_create(comp_ctx))
+ goto fail;
+
+#ifndef OS_ENABLE_HW_BOUND_CHECK
+ comp_ctx->enable_bound_check = true;
+ /* Always enable stack boundary check if `bounds-checks`
+ is enabled */
+ comp_ctx->enable_stack_bound_check = true;
+#else
+ comp_ctx->enable_bound_check = false;
+ /* When `bounds-checks` is disabled, we set stack boundary
+ check status according to the compilation option */
+#if WASM_DISABLE_STACK_HW_BOUND_CHECK != 0
+ /* Native stack overflow check with hardware trap is disabled,
+ we need to enable the check by LLVM JITed/AOTed code */
+ comp_ctx->enable_stack_bound_check = true;
+#else
+ /* Native stack overflow check with hardware trap is enabled,
+ no need to enable the check by LLVM JITed/AOTed code */
+ comp_ctx->enable_stack_bound_check = false;
+#endif
+#endif
+ }
+ else {
+ /* Create LLVM target machine */
+ arch = option->target_arch;
+ abi = option->target_abi;
+ cpu = option->target_cpu;
+ features = option->cpu_features;
+ opt_level = option->opt_level;
+ size_level = option->size_level;
+
+ /* verify external llc compiler */
+ comp_ctx->external_llc_compiler = getenv("WAMRC_LLC_COMPILER");
+ if (comp_ctx->external_llc_compiler) {
+#if defined(_WIN32) || defined(_WIN32_)
+ comp_ctx->external_llc_compiler = NULL;
+ LOG_WARNING("External LLC compiler not supported on Windows.");
+#else
+ if (access(comp_ctx->external_llc_compiler, X_OK) != 0) {
+ LOG_WARNING("WAMRC_LLC_COMPILER [%s] not found, fallback to "
+ "default pipeline",
+ comp_ctx->external_llc_compiler);
+ comp_ctx->external_llc_compiler = NULL;
+ }
+ else {
+ comp_ctx->llc_compiler_flags = getenv("WAMRC_LLC_FLAGS");
+ LOG_VERBOSE("Using external LLC compiler [%s]",
+ comp_ctx->external_llc_compiler);
+ }
+#endif
+ }
+
+ /* verify external asm compiler */
+ if (!comp_ctx->external_llc_compiler) {
+ comp_ctx->external_asm_compiler = getenv("WAMRC_ASM_COMPILER");
+ if (comp_ctx->external_asm_compiler) {
+#if defined(_WIN32) || defined(_WIN32_)
+ comp_ctx->external_asm_compiler = NULL;
+ LOG_WARNING("External ASM compiler not supported on Windows.");
+#else
+ if (access(comp_ctx->external_asm_compiler, X_OK) != 0) {
+ LOG_WARNING(
+ "WAMRC_ASM_COMPILER [%s] not found, fallback to "
+ "default pipeline",
+ comp_ctx->external_asm_compiler);
+ comp_ctx->external_asm_compiler = NULL;
+ }
+ else {
+ comp_ctx->asm_compiler_flags = getenv("WAMRC_ASM_FLAGS");
+ LOG_VERBOSE("Using external ASM compiler [%s]",
+ comp_ctx->external_asm_compiler);
+ }
+#endif
+ }
+ }
+
+ if (arch) {
+ /* Add default sub-arch if not specified */
+ if (!strcmp(arch, "arm"))
+ arch = "armv4";
+ else if (!strcmp(arch, "armeb"))
+ arch = "armv4eb";
+ else if (!strcmp(arch, "thumb"))
+ arch = "thumbv4t";
+ else if (!strcmp(arch, "thumbeb"))
+ arch = "thumbv4teb";
+ else if (!strcmp(arch, "aarch64"))
+ arch = "aarch64v8";
+ else if (!strcmp(arch, "aarch64_be"))
+ arch = "aarch64_bev8";
+ }
+
+ /* Check target arch */
+ if (arch && !check_target_arch(arch)) {
+ if (!strcmp(arch, "help"))
+ print_supported_targets();
+ else
+ aot_set_last_error(
+ "Invalid target. "
+ "Use --target=help to list all supported targets");
+ goto fail;
+ }
+
+ /* Check target ABI */
+ if (abi && !check_target_abi(abi)) {
+ if (!strcmp(abi, "help"))
+ print_supported_abis();
+ else
+ aot_set_last_error(
+ "Invalid target ABI. "
+ "Use --target-abi=help to list all supported ABI");
+ goto fail;
+ }
+
+ /* Set default abi for riscv target */
+ if (arch && !strncmp(arch, "riscv", 5) && !abi) {
+ if (!strcmp(arch, "riscv64"))
+ abi = "lp64d";
+ else
+ abi = "ilp32d";
+ }
+
+#if defined(__APPLE__) || defined(__MACH__)
+ if (!abi) {
+ /* On MacOS platform, set abi to "gnu" to avoid generating
+ object file of Mach-O binary format which is unsupported */
+ abi = "gnu";
+ if (!arch && !cpu && !features) {
+ /* Get CPU name of the host machine to avoid checking
+ SIMD capability failed */
+ if (!(cpu = cpu_new = LLVMGetHostCPUName())) {
+ aot_set_last_error("llvm get host cpu name failed.");
+ goto fail;
+ }
+ }
+ }
+#endif
+
+ if (abi) {
+ /* Construct target triple: <arch>-<vendor>-<sys>-<abi> */
+ const char *vendor_sys;
+ char *arch1 = arch, default_arch[32] = { 0 };
+
+ if (!arch1) {
+ char *default_triple = LLVMGetDefaultTargetTriple();
+
+ if (!default_triple) {
+ aot_set_last_error(
+ "llvm get default target triple failed.");
+ goto fail;
+ }
+
+ vendor_sys = strstr(default_triple, "-");
+ bh_assert(vendor_sys);
+ bh_memcpy_s(default_arch, sizeof(default_arch), default_triple,
+ (uint32)(vendor_sys - default_triple));
+ arch1 = default_arch;
+
+ LLVMDisposeMessage(default_triple);
+ }
+
+ /**
+ * Set <vendor>-<sys> according to abi to generate the object file
+ * with the correct file format which might be different from the
+ * default object file format of the host, e.g., generating AOT file
+ * for Windows/MacOS under Linux host, or generating AOT file for
+ * Linux/MacOS under Windows host.
+ */
+ if (!strcmp(abi, "msvc")) {
+ if (!strcmp(arch1, "i386"))
+ vendor_sys = "-pc-win32-";
+ else
+ vendor_sys = "-pc-windows-";
+ }
+ else {
+ vendor_sys = "-pc-linux-";
+ }
+
+ bh_assert(strlen(arch1) + strlen(vendor_sys) + strlen(abi)
+ < sizeof(triple_buf));
+ bh_memcpy_s(triple_buf, (uint32)sizeof(triple_buf), arch1,
+ (uint32)strlen(arch1));
+ bh_memcpy_s(triple_buf + strlen(arch1),
+ (uint32)(sizeof(triple_buf) - strlen(arch1)),
+ vendor_sys, (uint32)strlen(vendor_sys));
+ bh_memcpy_s(triple_buf + strlen(arch1) + strlen(vendor_sys),
+ (uint32)(sizeof(triple_buf) - strlen(arch1)
+ - strlen(vendor_sys)),
+ abi, (uint32)strlen(abi));
+ triple = triple_buf;
+ }
+ else if (arch) {
+ /* Construct target triple: <arch>-<vendor>-<sys>-<abi> */
+ const char *vendor_sys;
+ char *default_triple = LLVMGetDefaultTargetTriple();
+
+ if (!default_triple) {
+ aot_set_last_error("llvm get default target triple failed.");
+ goto fail;
+ }
+
+ if (strstr(default_triple, "windows")) {
+ vendor_sys = "-pc-windows-";
+ if (!abi)
+ abi = "msvc";
+ }
+ else if (strstr(default_triple, "win32")) {
+ vendor_sys = "-pc-win32-";
+ if (!abi)
+ abi = "msvc";
+ }
+ else {
+ vendor_sys = "-pc-linux-";
+ if (!abi)
+ abi = "gnu";
+ }
+
+ LLVMDisposeMessage(default_triple);
+
+ bh_assert(strlen(arch) + strlen(vendor_sys) + strlen(abi)
+ < sizeof(triple_buf));
+ bh_memcpy_s(triple_buf, (uint32)sizeof(triple_buf), arch,
+ (uint32)strlen(arch));
+ bh_memcpy_s(triple_buf + strlen(arch),
+ (uint32)(sizeof(triple_buf) - strlen(arch)), vendor_sys,
+ (uint32)strlen(vendor_sys));
+ bh_memcpy_s(triple_buf + strlen(arch) + strlen(vendor_sys),
+ (uint32)(sizeof(triple_buf) - strlen(arch)
+ - strlen(vendor_sys)),
+ abi, (uint32)strlen(abi));
+ triple = triple_buf;
+ }
+
+ if (!cpu && features) {
+ aot_set_last_error("cpu isn't specified for cpu features.");
+ goto fail;
+ }
+
+ if (!triple && !cpu) {
+ /* Get a triple for the host machine */
+ if (!(triple_norm = triple_norm_new =
+ LLVMGetDefaultTargetTriple())) {
+ aot_set_last_error("llvm get default target triple failed.");
+ goto fail;
+ }
+ /* Get CPU name of the host machine */
+ if (!(cpu = cpu_new = LLVMGetHostCPUName())) {
+ aot_set_last_error("llvm get host cpu name failed.");
+ goto fail;
+ }
+ }
+ else if (triple) {
+ /* Normalize a target triple */
+ if (!(triple_norm = triple_norm_new =
+ LLVMNormalizeTargetTriple(triple))) {
+ snprintf(buf, sizeof(buf),
+ "llvm normlalize target triple (%s) failed.", triple);
+ aot_set_last_error(buf);
+ goto fail;
+ }
+ if (!cpu)
+ cpu = "";
+ }
+ else {
+ /* triple is NULL, cpu isn't NULL */
+ snprintf(buf, sizeof(buf), "target isn't specified for cpu %s.",
+ cpu);
+ aot_set_last_error(buf);
+ goto fail;
+ }
+
+ /* Add module flag and cpu feature for riscv target */
+ if (arch && !strncmp(arch, "riscv", 5)) {
+ LLVMMetadataRef meta_target_abi;
+
+ if (!(meta_target_abi = LLVMMDStringInContext2(comp_ctx->context,
+ abi, strlen(abi)))) {
+ aot_set_last_error("create metadata string failed.");
+ goto fail;
+ }
+ LLVMAddModuleFlag(comp_ctx->module, LLVMModuleFlagBehaviorError,
+ "target-abi", strlen("target-abi"),
+ meta_target_abi);
+
+ if (!strcmp(abi, "lp64d") || !strcmp(abi, "ilp32d")) {
+ if (features) {
+ snprintf(features_buf, sizeof(features_buf), "%s%s",
+ features, ",+d");
+ features = features_buf;
+ }
+ else
+ features = "+d";
+ }
+ }
+
+ if (!features)
+ features = "";
+
+ /* Get target with triple, note that LLVMGetTargetFromTriple()
+ return 0 when success, but not true. */
+ if (LLVMGetTargetFromTriple(triple_norm, &target, &err) != 0) {
+ if (err) {
+ LLVMDisposeMessage(err);
+ err = NULL;
+ }
+ snprintf(buf, sizeof(buf),
+ "llvm get target from triple (%s) failed", triple_norm);
+ aot_set_last_error(buf);
+ goto fail;
+ }
+
+ /* Save target arch */
+ get_target_arch_from_triple(triple_norm, comp_ctx->target_arch,
+ sizeof(comp_ctx->target_arch));
+
+ if (option->bounds_checks == 1 || option->bounds_checks == 0) {
+ /* Set by user */
+ comp_ctx->enable_bound_check =
+ (option->bounds_checks == 1) ? true : false;
+ }
+ else {
+ /* Unset by user, use default value */
+ if (strstr(comp_ctx->target_arch, "64")
+ && !option->is_sgx_platform) {
+ comp_ctx->enable_bound_check = false;
+ }
+ else {
+ comp_ctx->enable_bound_check = true;
+ }
+ }
+
+ if (comp_ctx->enable_bound_check) {
+ /* Always enable stack boundary check if `bounds-checks`
+ is enabled */
+ comp_ctx->enable_stack_bound_check = true;
+ }
+ else {
+ /* When `bounds-checks` is disabled, we set stack boundary
+ check status according to the input option */
+ comp_ctx->enable_stack_bound_check =
+ (option->stack_bounds_checks == 1) ? true : false;
+ }
+
+ os_printf("Create AoT compiler with:\n");
+ os_printf(" target: %s\n", comp_ctx->target_arch);
+ os_printf(" target cpu: %s\n", cpu);
+ os_printf(" cpu features: %s\n", features);
+ os_printf(" opt level: %d\n", opt_level);
+ os_printf(" size level: %d\n", size_level);
+ switch (option->output_format) {
+ case AOT_LLVMIR_UNOPT_FILE:
+ os_printf(" output format: unoptimized LLVM IR\n");
+ break;
+ case AOT_LLVMIR_OPT_FILE:
+ os_printf(" output format: optimized LLVM IR\n");
+ break;
+ case AOT_FORMAT_FILE:
+ os_printf(" output format: AoT file\n");
+ break;
+ case AOT_OBJECT_FILE:
+ os_printf(" output format: native object file\n");
+ break;
+ }
+
+ if (!LLVMTargetHasTargetMachine(target)) {
+ snprintf(buf, sizeof(buf),
+ "no target machine for this target (%s).", triple_norm);
+ aot_set_last_error(buf);
+ goto fail;
+ }
+
+ /* Report error if target isn't arc and hasn't asm backend.
+ For arc target, as it cannot emit to memory buffer of elf file
+ currently, we let it emit to assembly file instead, and then call
+ arc-gcc to compile
+ asm file to elf file, and read elf file to memory buffer. */
+ if (strncmp(comp_ctx->target_arch, "arc", 3)
+ && !LLVMTargetHasAsmBackend(target)) {
+ snprintf(buf, sizeof(buf), "no asm backend for this target (%s).",
+ LLVMGetTargetName(target));
+ aot_set_last_error(buf);
+ goto fail;
+ }
+
+ /* Set code model */
+ if (size_level == 0)
+ code_model = LLVMCodeModelLarge;
+ else if (size_level == 1)
+ code_model = LLVMCodeModelMedium;
+ else if (size_level == 2)
+ code_model = LLVMCodeModelKernel;
+ else
+ code_model = LLVMCodeModelSmall;
+
+ /* Create the target machine */
+ if (!(comp_ctx->target_machine = LLVMCreateTargetMachineWithOpts(
+ target, triple_norm, cpu, features, opt_level,
+ LLVMRelocStatic, code_model, false,
+ option->stack_usage_file))) {
+ aot_set_last_error("create LLVM target machine failed.");
+ goto fail;
+ }
+ }
+
+ if (option->enable_simd && strcmp(comp_ctx->target_arch, "x86_64") != 0
+ && strncmp(comp_ctx->target_arch, "aarch64", 7) != 0) {
+ /* Disable simd if it isn't supported by target arch */
+ option->enable_simd = false;
+ }
+
+ if (option->enable_simd) {
+ char *tmp;
+ bool check_simd_ret;
+
+ comp_ctx->enable_simd = true;
+
+ if (!(tmp = LLVMGetTargetMachineCPU(comp_ctx->target_machine))) {
+ aot_set_last_error("get CPU from Target Machine fail");
+ goto fail;
+ }
+
+ check_simd_ret =
+ aot_check_simd_compatibility(comp_ctx->target_arch, tmp);
+ LLVMDisposeMessage(tmp);
+ if (!check_simd_ret) {
+ aot_set_last_error("SIMD compatibility check failed, "
+ "try adding --cpu=<cpu> to specify a cpu "
+ "or adding --disable-simd to disable SIMD");
+ goto fail;
+ }
+ }
+
+ if (!(target_data_ref =
+ LLVMCreateTargetDataLayout(comp_ctx->target_machine))) {
+ aot_set_last_error("create LLVM target data layout failed.");
+ goto fail;
+ }
+ comp_ctx->pointer_size = LLVMPointerSize(target_data_ref);
+ LLVMDisposeTargetData(target_data_ref);
+
+ comp_ctx->optimize = true;
+ if (option->output_format == AOT_LLVMIR_UNOPT_FILE)
+ comp_ctx->optimize = false;
+
+ /* Create metadata for llvm float experimental constrained intrinsics */
+ if (!(comp_ctx->fp_rounding_mode = LLVMMDStringInContext(
+ comp_ctx->context, fp_round, (uint32)strlen(fp_round)))
+ || !(comp_ctx->fp_exception_behavior = LLVMMDStringInContext(
+ comp_ctx->context, fp_exce, (uint32)strlen(fp_exce)))) {
+ aot_set_last_error("create float llvm metadata failed.");
+ goto fail;
+ }
+
+ if (!aot_set_llvm_basic_types(&comp_ctx->basic_types, comp_ctx->context)) {
+ aot_set_last_error("create LLVM basic types failed.");
+ goto fail;
+ }
+
+ if (!aot_create_llvm_consts(&comp_ctx->llvm_consts, comp_ctx)) {
+ aot_set_last_error("create LLVM const values failed.");
+ goto fail;
+ }
+
+ /* set exec_env data type to int8** */
+ comp_ctx->exec_env_type = comp_ctx->basic_types.int8_pptr_type;
+
+ /* set aot_inst data type to int8* */
+ comp_ctx->aot_inst_type = INT8_PTR_TYPE;
+
+ /* Create function context for each function */
+ comp_ctx->func_ctx_count = comp_data->func_count;
+ if (comp_data->func_count > 0
+ && !(comp_ctx->func_ctxes =
+ aot_create_func_contexts(comp_data, comp_ctx)))
+ goto fail;
+
+ if (cpu) {
+ uint32 len = (uint32)strlen(cpu) + 1;
+ if (!(comp_ctx->target_cpu = wasm_runtime_malloc(len))) {
+ aot_set_last_error("allocate memory failed");
+ goto fail;
+ }
+ bh_memcpy_s(comp_ctx->target_cpu, len, cpu, len);
+ }
+
+ if (comp_ctx->disable_llvm_intrinsics)
+ aot_intrinsic_fill_capability_flags(comp_ctx);
+
+ ret = comp_ctx;
+
+fail:
+ if (triple_norm_new)
+ LLVMDisposeMessage(triple_norm_new);
+
+ if (cpu_new)
+ LLVMDisposeMessage(cpu_new);
+
+ if (!ret)
+ aot_destroy_comp_context(comp_ctx);
+
+ (void)i;
+ return ret;
+}
+
+void
+aot_destroy_comp_context(AOTCompContext *comp_ctx)
+{
+ if (!comp_ctx)
+ return;
+
+ if (comp_ctx->target_machine)
+ LLVMDisposeTargetMachine(comp_ctx->target_machine);
+
+ if (comp_ctx->builder)
+ LLVMDisposeBuilder(comp_ctx->builder);
+
+ if (comp_ctx->orc_thread_safe_context)
+ LLVMOrcDisposeThreadSafeContext(comp_ctx->orc_thread_safe_context);
+
+ /* Note: don't dispose comp_ctx->context and comp_ctx->module as
+ they are disposed when disposing the thread safe context */
+
+ /* Has to be the last one */
+ if (comp_ctx->orc_jit)
+ LLVMOrcDisposeLLLazyJIT(comp_ctx->orc_jit);
+
+ if (comp_ctx->func_ctxes)
+ aot_destroy_func_contexts(comp_ctx->func_ctxes,
+ comp_ctx->func_ctx_count);
+
+ if (bh_list_length(&comp_ctx->native_symbols) > 0) {
+ AOTNativeSymbol *sym = bh_list_first_elem(&comp_ctx->native_symbols);
+ while (sym) {
+ AOTNativeSymbol *t = bh_list_elem_next(sym);
+ bh_list_remove(&comp_ctx->native_symbols, sym);
+ wasm_runtime_free(sym);
+ sym = t;
+ }
+ }
+
+ if (comp_ctx->target_cpu) {
+ wasm_runtime_free(comp_ctx->target_cpu);
+ }
+
+ wasm_runtime_free(comp_ctx);
+}
+
+static bool
+insert_native_symbol(AOTCompContext *comp_ctx, const char *symbol, int32 idx)
+{
+ AOTNativeSymbol *sym = wasm_runtime_malloc(sizeof(AOTNativeSymbol));
+
+ if (!sym) {
+ aot_set_last_error("alloc native symbol failed.");
+ return false;
+ }
+
+ memset(sym, 0, sizeof(AOTNativeSymbol));
+ bh_assert(strlen(symbol) <= sizeof(sym->symbol));
+ snprintf(sym->symbol, sizeof(sym->symbol), "%s", symbol);
+ sym->index = idx;
+
+ if (BH_LIST_ERROR == bh_list_insert(&comp_ctx->native_symbols, sym)) {
+ wasm_runtime_free(sym);
+ aot_set_last_error("insert native symbol to list failed.");
+ return false;
+ }
+
+ return true;
+}
+
+int32
+aot_get_native_symbol_index(AOTCompContext *comp_ctx, const char *symbol)
+{
+ int32 idx = -1;
+ AOTNativeSymbol *sym = NULL;
+
+ sym = bh_list_first_elem(&comp_ctx->native_symbols);
+
+ /* Lookup an existing symobl record */
+
+ while (sym) {
+ if (strcmp(sym->symbol, symbol) == 0) {
+ idx = sym->index;
+ break;
+ }
+ sym = bh_list_elem_next(sym);
+ }
+
+ /* Given symbol is not exist in list, then we alloc a new index for it */
+
+ if (idx < 0) {
+ if (comp_ctx->pointer_size == sizeof(uint32)
+ && (!strncmp(symbol, "f64#", 4) || !strncmp(symbol, "i64#", 4))) {
+ idx = bh_list_length(&comp_ctx->native_symbols);
+ /* Add 4 bytes padding on 32-bit target to make sure that
+ the f64 const is stored on 8-byte aligned address */
+ if (idx & 1) {
+ if (!insert_native_symbol(comp_ctx, "__ignore", idx)) {
+ return -1;
+ }
+ }
+ }
+
+ idx = bh_list_length(&comp_ctx->native_symbols);
+ if (!insert_native_symbol(comp_ctx, symbol, idx)) {
+ return -1;
+ }
+
+ if (comp_ctx->pointer_size == sizeof(uint32)
+ && (!strncmp(symbol, "f64#", 4) || !strncmp(symbol, "i64#", 4))) {
+ /* f64 const occupies 2 pointer slots on 32-bit target */
+ if (!insert_native_symbol(comp_ctx, "__ignore", idx + 1)) {
+ return -1;
+ }
+ }
+ }
+
+ return idx;
+}
+
+void
+aot_value_stack_push(AOTValueStack *stack, AOTValue *value)
+{
+ if (!stack->value_list_head)
+ stack->value_list_head = stack->value_list_end = value;
+ else {
+ stack->value_list_end->next = value;
+ value->prev = stack->value_list_end;
+ stack->value_list_end = value;
+ }
+}
+
+AOTValue *
+aot_value_stack_pop(AOTValueStack *stack)
+{
+ AOTValue *value = stack->value_list_end;
+
+ bh_assert(stack->value_list_end);
+
+ if (stack->value_list_head == stack->value_list_end)
+ stack->value_list_head = stack->value_list_end = NULL;
+ else {
+ stack->value_list_end = stack->value_list_end->prev;
+ stack->value_list_end->next = NULL;
+ value->prev = NULL;
+ }
+
+ return value;
+}
+
+void
+aot_value_stack_destroy(AOTValueStack *stack)
+{
+ AOTValue *value = stack->value_list_head, *p;
+
+ while (value) {
+ p = value->next;
+ wasm_runtime_free(value);
+ value = p;
+ }
+
+ stack->value_list_head = NULL;
+ stack->value_list_end = NULL;
+}
+
+void
+aot_block_stack_push(AOTBlockStack *stack, AOTBlock *block)
+{
+ if (!stack->block_list_head)
+ stack->block_list_head = stack->block_list_end = block;
+ else {
+ stack->block_list_end->next = block;
+ block->prev = stack->block_list_end;
+ stack->block_list_end = block;
+ }
+}
+
+AOTBlock *
+aot_block_stack_pop(AOTBlockStack *stack)
+{
+ AOTBlock *block = stack->block_list_end;
+
+ bh_assert(stack->block_list_end);
+
+ if (stack->block_list_head == stack->block_list_end)
+ stack->block_list_head = stack->block_list_end = NULL;
+ else {
+ stack->block_list_end = stack->block_list_end->prev;
+ stack->block_list_end->next = NULL;
+ block->prev = NULL;
+ }
+
+ return block;
+}
+
+void
+aot_block_stack_destroy(AOTBlockStack *stack)
+{
+ AOTBlock *block = stack->block_list_head, *p;
+
+ while (block) {
+ p = block->next;
+ aot_value_stack_destroy(&block->value_stack);
+ aot_block_destroy(block);
+ block = p;
+ }
+
+ stack->block_list_head = NULL;
+ stack->block_list_end = NULL;
+}
+
+void
+aot_block_destroy(AOTBlock *block)
+{
+ aot_value_stack_destroy(&block->value_stack);
+ if (block->param_types)
+ wasm_runtime_free(block->param_types);
+ if (block->param_phis)
+ wasm_runtime_free(block->param_phis);
+ if (block->else_param_phis)
+ wasm_runtime_free(block->else_param_phis);
+ if (block->result_types)
+ wasm_runtime_free(block->result_types);
+ if (block->result_phis)
+ wasm_runtime_free(block->result_phis);
+ wasm_runtime_free(block);
+}
+
+bool
+aot_checked_addr_list_add(AOTFuncContext *func_ctx, uint32 local_idx,
+ uint32 offset, uint32 bytes)
+{
+ AOTCheckedAddr *node = func_ctx->checked_addr_list;
+
+ if (!(node = wasm_runtime_malloc(sizeof(AOTCheckedAddr)))) {
+ aot_set_last_error("allocate memory failed.");
+ return false;
+ }
+
+ node->local_idx = local_idx;
+ node->offset = offset;
+ node->bytes = bytes;
+
+ node->next = func_ctx->checked_addr_list;
+ func_ctx->checked_addr_list = node;
+ return true;
+}
+
+void
+aot_checked_addr_list_del(AOTFuncContext *func_ctx, uint32 local_idx)
+{
+ AOTCheckedAddr *node = func_ctx->checked_addr_list;
+ AOTCheckedAddr *node_prev = NULL, *node_next;
+
+ while (node) {
+ node_next = node->next;
+
+ if (node->local_idx == local_idx) {
+ if (!node_prev)
+ func_ctx->checked_addr_list = node_next;
+ else
+ node_prev->next = node_next;
+ wasm_runtime_free(node);
+ }
+ else {
+ node_prev = node;
+ }
+
+ node = node_next;
+ }
+}
+
+bool
+aot_checked_addr_list_find(AOTFuncContext *func_ctx, uint32 local_idx,
+ uint32 offset, uint32 bytes)
+{
+ AOTCheckedAddr *node = func_ctx->checked_addr_list;
+
+ while (node) {
+ if (node->local_idx == local_idx && node->offset == offset
+ && node->bytes >= bytes) {
+ return true;
+ }
+ node = node->next;
+ }
+
+ return false;
+}
+
+void
+aot_checked_addr_list_destroy(AOTFuncContext *func_ctx)
+{
+ AOTCheckedAddr *node = func_ctx->checked_addr_list, *node_next;
+
+ while (node) {
+ node_next = node->next;
+ wasm_runtime_free(node);
+ node = node_next;
+ }
+
+ func_ctx->checked_addr_list = NULL;
+}
+
+bool
+aot_build_zero_function_ret(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
+ AOTFuncType *func_type)
+{
+ LLVMValueRef ret = NULL;
+
+ if (func_type->result_count) {
+ switch (func_type->types[func_type->param_count]) {
+ case VALUE_TYPE_I32:
+ ret = LLVMBuildRet(comp_ctx->builder, I32_ZERO);
+ break;
+ case VALUE_TYPE_I64:
+ ret = LLVMBuildRet(comp_ctx->builder, I64_ZERO);
+ break;
+ case VALUE_TYPE_F32:
+ ret = LLVMBuildRet(comp_ctx->builder, F32_ZERO);
+ break;
+ case VALUE_TYPE_F64:
+ ret = LLVMBuildRet(comp_ctx->builder, F64_ZERO);
+ break;
+ case VALUE_TYPE_V128:
+ ret =
+ LLVMBuildRet(comp_ctx->builder, LLVM_CONST(i64x2_vec_zero));
+ break;
+ case VALUE_TYPE_FUNCREF:
+ case VALUE_TYPE_EXTERNREF:
+ ret = LLVMBuildRet(comp_ctx->builder, REF_NULL);
+ break;
+ default:
+ bh_assert(0);
+ }
+ }
+ else {
+ ret = LLVMBuildRetVoid(comp_ctx->builder);
+ }
+
+ if (!ret) {
+ aot_set_last_error("llvm build ret failed.");
+ return false;
+ }
+#if WASM_ENABLE_DEBUG_AOT != 0
+ LLVMMetadataRef return_location =
+ dwarf_gen_func_ret_location(comp_ctx, func_ctx);
+ LLVMInstructionSetDebugLoc(ret, return_location);
+#endif
+ return true;
+}
+
+static LLVMValueRef
+__call_llvm_intrinsic(const AOTCompContext *comp_ctx,
+ const AOTFuncContext *func_ctx, const char *name,
+ LLVMTypeRef ret_type, LLVMTypeRef *param_types,
+ int param_count, LLVMValueRef *param_values)
+{
+ LLVMValueRef func, ret;
+ LLVMTypeRef func_type;
+ const char *symname;
+ int32 func_idx;
+
+ if (comp_ctx->disable_llvm_intrinsics
+ && aot_intrinsic_check_capability(comp_ctx, name)) {
+ if (func_ctx == NULL) {
+ aot_set_last_error_v("invalid func_ctx for intrinsic: %s", name);
+ return NULL;
+ }
+
+ if (!(func_type = LLVMFunctionType(ret_type, param_types,
+ (uint32)param_count, false))) {
+ aot_set_last_error("create LLVM intrinsic function type failed.");
+ return NULL;
+ }
+ if (!(func_type = LLVMPointerType(func_type, 0))) {
+ aot_set_last_error(
+ "create LLVM intrinsic function pointer type failed.");
+ return NULL;
+ }
+
+ if (!(symname = aot_intrinsic_get_symbol(name))) {
+ aot_set_last_error_v("runtime intrinsic not implemented: %s\n",
+ name);
+ return NULL;
+ }
+
+ func_idx =
+ aot_get_native_symbol_index((AOTCompContext *)comp_ctx, symname);
+ if (func_idx < 0) {
+ aot_set_last_error_v("get runtime intrinsc index failed: %s\n",
+ name);
+ return NULL;
+ }
+
+ if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol,
+ func_type, func_idx))) {
+ aot_set_last_error_v("get runtime intrinsc failed: %s\n", name);
+ return NULL;
+ }
+ }
+ else {
+ /* Declare llvm intrinsic function if necessary */
+ if (!(func = LLVMGetNamedFunction(func_ctx->module, name))) {
+ if (!(func_type = LLVMFunctionType(ret_type, param_types,
+ (uint32)param_count, false))) {
+ aot_set_last_error(
+ "create LLVM intrinsic function type failed.");
+ return NULL;
+ }
+
+ if (!(func = LLVMAddFunction(func_ctx->module, name, func_type))) {
+ aot_set_last_error("add LLVM intrinsic function failed.");
+ return NULL;
+ }
+ }
+ }
+
+#if LLVM_VERSION_MAJOR >= 14
+ func_type =
+ LLVMFunctionType(ret_type, param_types, (uint32)param_count, false);
+#endif
+
+ /* Call the LLVM intrinsic function */
+ if (!(ret = LLVMBuildCall2(comp_ctx->builder, func_type, func, param_values,
+ (uint32)param_count, "call"))) {
+ aot_set_last_error("llvm build intrinsic call failed.");
+ return NULL;
+ }
+
+ return ret;
+}
+
+LLVMValueRef
+aot_call_llvm_intrinsic(const AOTCompContext *comp_ctx,
+ const AOTFuncContext *func_ctx, const char *intrinsic,
+ LLVMTypeRef ret_type, LLVMTypeRef *param_types,
+ int param_count, ...)
+{
+ LLVMValueRef *param_values, ret;
+ va_list argptr;
+ uint64 total_size;
+ int i = 0;
+
+ /* Create param values */
+ total_size = sizeof(LLVMValueRef) * (uint64)param_count;
+ if (total_size >= UINT32_MAX
+ || !(param_values = wasm_runtime_malloc((uint32)total_size))) {
+ aot_set_last_error("allocate memory for param values failed.");
+ return false;
+ }
+
+ /* Load each param value */
+ va_start(argptr, param_count);
+ while (i < param_count)
+ param_values[i++] = va_arg(argptr, LLVMValueRef);
+ va_end(argptr);
+
+ ret = __call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, ret_type,
+ param_types, param_count, param_values);
+
+ wasm_runtime_free(param_values);
+
+ return ret;
+}
+
+LLVMValueRef
+aot_call_llvm_intrinsic_v(const AOTCompContext *comp_ctx,
+ const AOTFuncContext *func_ctx, const char *intrinsic,
+ LLVMTypeRef ret_type, LLVMTypeRef *param_types,
+ int param_count, va_list param_value_list)
+{
+ LLVMValueRef *param_values, ret;
+ uint64 total_size;
+ int i = 0;
+
+ /* Create param values */
+ total_size = sizeof(LLVMValueRef) * (uint64)param_count;
+ if (total_size >= UINT32_MAX
+ || !(param_values = wasm_runtime_malloc((uint32)total_size))) {
+ aot_set_last_error("allocate memory for param values failed.");
+ return false;
+ }
+
+ /* Load each param value */
+ while (i < param_count)
+ param_values[i++] = va_arg(param_value_list, LLVMValueRef);
+
+ ret = __call_llvm_intrinsic(comp_ctx, func_ctx, intrinsic, ret_type,
+ param_types, param_count, param_values);
+
+ wasm_runtime_free(param_values);
+
+ return ret;
+}
+
+LLVMValueRef
+aot_get_func_from_table(const AOTCompContext *comp_ctx, LLVMValueRef base,
+ LLVMTypeRef func_type, int32 index)
+{
+ LLVMValueRef func;
+ LLVMValueRef func_addr;
+
+ if (!(func_addr = I32_CONST(index))) {
+ aot_set_last_error("construct function index failed.");
+ goto fail;
+ }
+
+ if (!(func_addr =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE, base,
+ &func_addr, 1, "func_addr"))) {
+ aot_set_last_error("get function addr by index failed.");
+ goto fail;
+ }
+
+ func =
+ LLVMBuildLoad2(comp_ctx->builder, OPQ_PTR_TYPE, func_addr, "func_tmp");
+
+ if (func == NULL) {
+ aot_set_last_error("get function pointer failed.");
+ goto fail;
+ }
+
+ if (!(func =
+ LLVMBuildBitCast(comp_ctx->builder, func, func_type, "func"))) {
+ aot_set_last_error("cast function fialed.");
+ goto fail;
+ }
+
+ return func;
+fail:
+ return NULL;
+}
+
+LLVMValueRef
+aot_load_const_from_table(AOTCompContext *comp_ctx, LLVMValueRef base,
+ const WASMValue *value, uint8 value_type)
+{
+ LLVMValueRef const_index, const_addr, const_value;
+ LLVMTypeRef const_ptr_type, const_type;
+ char buf[128] = { 0 };
+ int32 index;
+
+ switch (value_type) {
+ case VALUE_TYPE_I32:
+ /* Store the raw int bits of i32 const as a hex string */
+ snprintf(buf, sizeof(buf), "i32#%08" PRIX32, value->i32);
+ const_ptr_type = INT32_PTR_TYPE;
+ const_type = I32_TYPE;
+ break;
+ case VALUE_TYPE_I64:
+ /* Store the raw int bits of i64 const as a hex string */
+ snprintf(buf, sizeof(buf), "i64#%016" PRIX64, value->i64);
+ const_ptr_type = INT64_PTR_TYPE;
+ const_type = I64_TYPE;
+ break;
+ case VALUE_TYPE_F32:
+ /* Store the raw int bits of f32 const as a hex string */
+ snprintf(buf, sizeof(buf), "f32#%08" PRIX32, value->i32);
+ const_ptr_type = F32_PTR_TYPE;
+ const_type = F32_TYPE;
+ break;
+ case VALUE_TYPE_F64:
+ /* Store the raw int bits of f64 const as a hex string */
+ snprintf(buf, sizeof(buf), "f64#%016" PRIX64, value->i64);
+ const_ptr_type = F64_PTR_TYPE;
+ const_type = F64_TYPE;
+ break;
+ default:
+ bh_assert(0);
+ return NULL;
+ }
+
+ /* Load f32/f64 const from exec_env->native_symbol[index] */
+
+ index = aot_get_native_symbol_index(comp_ctx, buf);
+ if (index < 0) {
+ return NULL;
+ }
+
+ if (!(const_index = I32_CONST(index))) {
+ aot_set_last_error("construct const index failed.");
+ return NULL;
+ }
+
+ if (!(const_addr =
+ LLVMBuildInBoundsGEP2(comp_ctx->builder, OPQ_PTR_TYPE, base,
+ &const_index, 1, "const_addr_tmp"))) {
+ aot_set_last_error("get const addr by index failed.");
+ return NULL;
+ }
+
+ if (!(const_addr = LLVMBuildBitCast(comp_ctx->builder, const_addr,
+ const_ptr_type, "const_addr"))) {
+ aot_set_last_error("cast const fialed.");
+ return NULL;
+ }
+
+ if (!(const_value = LLVMBuildLoad2(comp_ctx->builder, const_type,
+ const_addr, "const_value"))) {
+ aot_set_last_error("load const failed.");
+ return NULL;
+ }
+
+ (void)const_type;
+ return const_value;
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-31 00:14:45 +0000