diff options
Diffstat (limited to 'fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_emit_control.c')
| -rw-r--r-- | fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_emit_control.c | 1155 |
1 files changed, 1155 insertions, 0 deletions
diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_emit_control.c b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_emit_control.c new file mode 100644 index 000000000..2cf51cf67 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/compilation/aot_emit_control.c @@ -0,0 +1,1155 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "aot_emit_control.h" +#include "aot_emit_exception.h" +#include "../aot/aot_runtime.h" +#include "../interpreter/wasm_loader.h" + +#if WASM_ENABLE_DEBUG_AOT != 0 +#include "debug/dwarf_extractor.h" +#endif + +static char *block_name_prefix[] = { "block", "loop", "if" }; +static char *block_name_suffix[] = { "begin", "else", "end" }; + +/* clang-format off */ +enum { + LABEL_BEGIN = 0, + LABEL_ELSE, + LABEL_END +}; +/* clang-format on */ + +static void +format_block_name(char *name, uint32 name_size, uint32 block_index, + uint32 label_type, uint32 label_id) +{ + if (label_type != LABEL_TYPE_FUNCTION) + snprintf(name, name_size, "%s%d%s%s", block_name_prefix[label_type], + block_index, "_", block_name_suffix[label_id]); + else + snprintf(name, name_size, "%s", "func_end"); +} + +#define CREATE_BLOCK(new_llvm_block, name) \ + do { \ + if (!(new_llvm_block = LLVMAppendBasicBlockInContext( \ + comp_ctx->context, func_ctx->func, name))) { \ + aot_set_last_error("add LLVM basic block failed."); \ + goto fail; \ + } \ + } while (0) + +#define CURR_BLOCK() LLVMGetInsertBlock(comp_ctx->builder) + +#define MOVE_BLOCK_AFTER(llvm_block, llvm_block_after) \ + LLVMMoveBasicBlockAfter(llvm_block, llvm_block_after) + +#define MOVE_BLOCK_AFTER_CURR(llvm_block) \ + LLVMMoveBasicBlockAfter(llvm_block, CURR_BLOCK()) + +#define MOVE_BLOCK_BEFORE(llvm_block, llvm_block_before) \ + LLVMMoveBasicBlockBefore(llvm_block, llvm_block_before) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define SET_BUILDER_POS(llvm_block) \ + LLVMPositionBuilderAtEnd(comp_ctx->builder, llvm_block) + +#define CREATE_RESULT_VALUE_PHIS(block) \ + do { \ + if (block->result_count && !block->result_phis) { \ + uint32 _i; \ + uint64 _size; \ + LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \ + /* Allocate memory */ \ + _size = sizeof(LLVMValueRef) * (uint64)block->result_count; \ + if (_size >= UINT32_MAX \ + || !(block->result_phis = \ + wasm_runtime_malloc((uint32)_size))) { \ + aot_set_last_error("allocate memory failed."); \ + goto fail; \ + } \ + SET_BUILDER_POS(block->llvm_end_block); \ + for (_i = 0; _i < block->result_count; _i++) { \ + if (!(block->result_phis[_i] = LLVMBuildPhi( \ + comp_ctx->builder, \ + TO_LLVM_TYPE(block->result_types[_i]), "phi"))) { \ + aot_set_last_error("llvm build phi failed."); \ + goto fail; \ + } \ + } \ + SET_BUILDER_POS(_block_curr); \ + } \ + } while (0) + +#define ADD_TO_RESULT_PHIS(block, value, idx) \ + do { \ + LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \ + LLVMTypeRef phi_ty = LLVMTypeOf(block->result_phis[idx]); \ + LLVMTypeRef value_ty = LLVMTypeOf(value); \ + bh_assert(LLVMGetTypeKind(phi_ty) == LLVMGetTypeKind(value_ty)); \ + bh_assert(LLVMGetTypeContext(phi_ty) == LLVMGetTypeContext(value_ty)); \ + LLVMAddIncoming(block->result_phis[idx], &value, &_block_curr, 1); \ + (void)phi_ty; \ + (void)value_ty; \ + } while (0) + +#define BUILD_ICMP(op, left, right, res, name) \ + do { \ + if (!(res = \ + LLVMBuildICmp(comp_ctx->builder, op, left, right, name))) { \ + aot_set_last_error("llvm build icmp failed."); \ + goto fail; \ + } \ + } while (0) + +#define ADD_TO_PARAM_PHIS(block, value, idx) \ + do { \ + LLVMBasicBlockRef _block_curr = CURR_BLOCK(); \ + LLVMAddIncoming(block->param_phis[idx], &value, &_block_curr, 1); \ + } while (0) + +static LLVMBasicBlockRef +find_next_llvm_end_block(AOTBlock *block) +{ + block = block->prev; + while (block && !block->llvm_end_block) + block = block->prev; + return block ? block->llvm_end_block : NULL; +} + +static AOTBlock * +get_target_block(AOTFuncContext *func_ctx, uint32 br_depth) +{ + uint32 i = br_depth; + AOTBlock *block = func_ctx->block_stack.block_list_end; + + while (i-- > 0 && block) { + block = block->prev; + } + + if (!block) { + aot_set_last_error("WASM block stack underflow."); + return NULL; + } + return block; +} + +static bool +handle_next_reachable_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block = func_ctx->block_stack.block_list_end; + AOTBlock *block_prev; + uint8 *frame_ip = NULL; + uint32 i; + AOTFuncType *func_type; + LLVMValueRef ret; +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMMetadataRef return_location; +#endif + + aot_checked_addr_list_destroy(func_ctx); + bh_assert(block); + +#if WASM_ENABLE_DEBUG_AOT != 0 + return_location = dwarf_gen_location( + comp_ctx, func_ctx, + (*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); +#endif + if (block->label_type == LABEL_TYPE_IF && block->llvm_else_block + && *p_frame_ip <= block->wasm_code_else) { + /* Clear value stack and start to translate else branch */ + aot_value_stack_destroy(&block->value_stack); + /* Recover parameters of else branch */ + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = block->wasm_code_else + 1; + return true; + } + + while (block && !block->is_reachable) { + block_prev = block->prev; + block = aot_block_stack_pop(&func_ctx->block_stack); + + if (block->label_type == LABEL_TYPE_IF) { + if (block->llvm_else_block && !block->skip_wasm_code_else + && *p_frame_ip <= block->wasm_code_else) { + /* Clear value stack and start to translate else branch */ + aot_value_stack_destroy(&block->value_stack); + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = block->wasm_code_else + 1; + /* Push back the block */ + aot_block_stack_push(&func_ctx->block_stack, block); + return true; + } + else if (block->llvm_end_block) { + /* Remove unreachable basic block */ + LLVMDeleteBasicBlock(block->llvm_end_block); + block->llvm_end_block = NULL; + } + } + + frame_ip = block->wasm_code_end; + aot_block_destroy(block); + block = block_prev; + } + + if (!block) { + *p_frame_ip = frame_ip + 1; + return true; + } + + *p_frame_ip = block->wasm_code_end + 1; + SET_BUILDER_POS(block->llvm_end_block); + + /* Pop block, push its return value, and destroy the block */ + block = aot_block_stack_pop(&func_ctx->block_stack); + func_type = func_ctx->aot_func->func_type; + for (i = 0; i < block->result_count; i++) { + bh_assert(block->result_phis[i]); + if (block->label_type != LABEL_TYPE_FUNCTION) { + PUSH(block->result_phis[i], block->result_types[i]); + } + else { + /* Store extra return values to function parameters */ + if (i != 0) { + uint32 param_index = func_type->param_count + i; + if (!LLVMBuildStore( + comp_ctx->builder, block->result_phis[i], + LLVMGetParam(func_ctx->func, param_index))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + } + } + } + if (block->label_type == LABEL_TYPE_FUNCTION) { + if (block->result_count) { + /* Return the first return value */ + if (!(ret = + LLVMBuildRet(comp_ctx->builder, block->result_phis[0]))) { + aot_set_last_error("llvm build return failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMInstructionSetDebugLoc(ret, return_location); +#endif + } + else { + if (!(ret = LLVMBuildRetVoid(comp_ctx->builder))) { + aot_set_last_error("llvm build return void failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMInstructionSetDebugLoc(ret, return_location); +#endif + } + } + aot_block_destroy(block); + return true; +fail: + return false; +} + +static bool +push_aot_block_to_stack_and_pass_params(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + AOTBlock *block) +{ + uint32 i, param_index; + LLVMValueRef value; + uint64 size; + char name[32]; + LLVMBasicBlockRef block_curr = CURR_BLOCK(); + + if (block->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block->param_count; + if (size >= UINT32_MAX + || !(block->param_phis = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + return false; + } + + if (block->label_type == LABEL_TYPE_IF && !block->skip_wasm_code_else + && !(block->else_param_phis = wasm_runtime_malloc((uint32)size))) { + wasm_runtime_free(block->param_phis); + block->param_phis = NULL; + aot_set_last_error("allocate memory failed."); + return false; + } + + /* Create param phis */ + for (i = 0; i < block->param_count; i++) { + SET_BUILDER_POS(block->llvm_entry_block); + snprintf(name, sizeof(name), "%s%d_phi%d", + block_name_prefix[block->label_type], block->block_index, + i); + if (!(block->param_phis[i] = LLVMBuildPhi( + comp_ctx->builder, TO_LLVM_TYPE(block->param_types[i]), + name))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else && block->llvm_else_block) { + /* Build else param phis */ + SET_BUILDER_POS(block->llvm_else_block); + snprintf(name, sizeof(name), "else%d_phi%d", block->block_index, + i); + if (!(block->else_param_phis[i] = LLVMBuildPhi( + comp_ctx->builder, + TO_LLVM_TYPE(block->param_types[i]), name))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + } + } + SET_BUILDER_POS(block_curr); + + /* Pop param values from current block's + * value stack and add to param phis. + */ + for (i = 0; i < block->param_count; i++) { + param_index = block->param_count - 1 - i; + POP(value, block->param_types[param_index]); + ADD_TO_PARAM_PHIS(block, value, param_index); + if (block->label_type == LABEL_TYPE_IF + && !block->skip_wasm_code_else) { + if (block->llvm_else_block) { + /* has else branch, add to else param phis */ + LLVMAddIncoming(block->else_param_phis[param_index], &value, + &block_curr, 1); + } + else { + /* no else branch, add to result phis */ + CREATE_RESULT_VALUE_PHIS(block); + ADD_TO_RESULT_PHIS(block, value, param_index); + } + } + } + } + + /* Push the new block to block stack */ + aot_block_stack_push(&func_ctx->block_stack, block); + + /* Push param phis to the new block */ + for (i = 0; i < block->param_count; i++) { + PUSH(block->param_phis[i], block->param_types[i]); + } + + return true; + +fail: + if (block->param_phis) { + wasm_runtime_free(block->param_phis); + block->param_phis = NULL; + } + if (block->else_param_phis) { + wasm_runtime_free(block->else_param_phis); + block->else_param_phis = NULL; + } + return false; +} + +bool +aot_compile_op_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip, uint8 *frame_ip_end, uint32 label_type, + uint32 param_count, uint8 *param_types, + uint32 result_count, uint8 *result_types) +{ + BlockAddr block_addr_cache[BLOCK_ADDR_CACHE_SIZE][BLOCK_ADDR_CONFLICT_SIZE]; + AOTBlock *block; + uint8 *else_addr, *end_addr; + LLVMValueRef value; + char name[32]; + + /* Check block stack */ + if (!func_ctx->block_stack.block_list_end) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + + memset(block_addr_cache, 0, sizeof(block_addr_cache)); + + /* Get block info */ + if (!(wasm_loader_find_block_addr( + NULL, (BlockAddr *)block_addr_cache, *p_frame_ip, frame_ip_end, + (uint8)label_type, &else_addr, &end_addr))) { + aot_set_last_error("find block end addr failed."); + return false; + } + + /* Allocate memory */ + if (!(block = wasm_runtime_malloc(sizeof(AOTBlock)))) { + aot_set_last_error("allocate memory failed."); + return false; + } + memset(block, 0, sizeof(AOTBlock)); + if (param_count + && !(block->param_types = wasm_runtime_malloc(param_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + if (result_count) { + if (!(block->result_types = wasm_runtime_malloc(result_count))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + } + + /* Init aot block data */ + block->label_type = label_type; + block->param_count = param_count; + if (param_count) { + bh_memcpy_s(block->param_types, param_count, param_types, param_count); + } + block->result_count = result_count; + if (result_count) { + bh_memcpy_s(block->result_types, result_count, result_types, + result_count); + } + block->wasm_code_else = else_addr; + block->wasm_code_end = end_addr; + block->block_index = func_ctx->block_stack.block_index[label_type]; + func_ctx->block_stack.block_index[label_type]++; + + if (label_type == LABEL_TYPE_BLOCK || label_type == LABEL_TYPE_LOOP) { + /* Create block */ + format_block_name(name, sizeof(name), block->block_index, label_type, + LABEL_BEGIN); + CREATE_BLOCK(block->llvm_entry_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); + /* Jump to the entry block */ + BUILD_BR(block->llvm_entry_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, block)) + goto fail; + /* Start to translate the block */ + SET_BUILDER_POS(block->llvm_entry_block); + if (label_type == LABEL_TYPE_LOOP) + aot_checked_addr_list_destroy(func_ctx); + } + else if (label_type == LABEL_TYPE_IF) { + POP_COND(value); + + if (LLVMIsUndef(value) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(value) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + aot_block_destroy(block); + return aot_handle_next_reachable_block(comp_ctx, func_ctx, + p_frame_ip); + } + + if (!LLVMIsConstant(value)) { + /* Compare value is not constant, create condition br IR */ + /* Create entry block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_BEGIN); + CREATE_BLOCK(block->llvm_entry_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); + + /* Create end block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_END); + CREATE_BLOCK(block->llvm_end_block, name); + MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_entry_block); + + if (else_addr) { + /* Create else block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_ELSE); + CREATE_BLOCK(block->llvm_else_block, name); + MOVE_BLOCK_AFTER(block->llvm_else_block, + block->llvm_entry_block); + /* Create condition br IR */ + BUILD_COND_BR(value, block->llvm_entry_block, + block->llvm_else_block); + } + else { + /* Create condition br IR */ + BUILD_COND_BR(value, block->llvm_entry_block, + block->llvm_end_block); + block->is_reachable = true; + } + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, + block)) + goto fail; + /* Start to translate if branch of BLOCK if */ + SET_BUILDER_POS(block->llvm_entry_block); + } + else { + if ((int32)LLVMConstIntGetZExtValue(value) != 0) { + /* Compare value is not 0, condition is true, else branch of + BLOCK if cannot be reached */ + block->skip_wasm_code_else = true; + /* Create entry block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_BEGIN); + CREATE_BLOCK(block->llvm_entry_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_entry_block); + /* Jump to the entry block */ + BUILD_BR(block->llvm_entry_block); + if (!push_aot_block_to_stack_and_pass_params(comp_ctx, func_ctx, + block)) + goto fail; + /* Start to translate the if branch */ + SET_BUILDER_POS(block->llvm_entry_block); + } + else { + /* Compare value is not 0, condition is false, if branch of + BLOCK if cannot be reached */ + if (else_addr) { + /* Create else block */ + format_block_name(name, sizeof(name), block->block_index, + label_type, LABEL_ELSE); + CREATE_BLOCK(block->llvm_else_block, name); + MOVE_BLOCK_AFTER_CURR(block->llvm_else_block); + /* Jump to the else block */ + BUILD_BR(block->llvm_else_block); + if (!push_aot_block_to_stack_and_pass_params( + comp_ctx, func_ctx, block)) + goto fail; + /* Start to translate the else branch */ + SET_BUILDER_POS(block->llvm_else_block); + *p_frame_ip = else_addr + 1; + } + else { + /* skip the block */ + aot_block_destroy(block); + *p_frame_ip = end_addr + 1; + } + } + } + } + else { + aot_set_last_error("Invalid block type."); + goto fail; + } + + return true; +fail: + aot_block_destroy(block); + return false; +} + +bool +aot_compile_op_else(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block = func_ctx->block_stack.block_list_end; + LLVMValueRef value; + char name[32]; + uint32 i, result_index; + + /* Check block */ + if (!block) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + if (block->label_type != LABEL_TYPE_IF + || (!block->skip_wasm_code_else && !block->llvm_else_block)) { + aot_set_last_error("Invalid WASM block type."); + return false; + } + + /* Create end block if needed */ + if (!block->llvm_end_block) { + format_block_name(name, sizeof(name), block->block_index, + block->label_type, LABEL_END); + CREATE_BLOCK(block->llvm_end_block, name); + if (block->llvm_else_block) + MOVE_BLOCK_AFTER(block->llvm_end_block, block->llvm_else_block); + else + MOVE_BLOCK_AFTER_CURR(block->llvm_end_block); + } + + block->is_reachable = true; + + /* Comes from the if branch of BLOCK if */ + CREATE_RESULT_VALUE_PHIS(block); + for (i = 0; i < block->result_count; i++) { + result_index = block->result_count - 1 - i; + POP(value, block->result_types[result_index]); + ADD_TO_RESULT_PHIS(block, value, result_index); + } + + /* Jump to end block */ + BUILD_BR(block->llvm_end_block); + + if (!block->skip_wasm_code_else && block->llvm_else_block) { + /* Clear value stack, recover param values + * and start to translate else branch. + */ + aot_value_stack_destroy(&block->value_stack); + for (i = 0; i < block->param_count; i++) + PUSH(block->else_param_phis[i], block->param_types[i]); + SET_BUILDER_POS(block->llvm_else_block); + aot_checked_addr_list_destroy(func_ctx); + return true; + } + + /* No else branch or no need to translate else branch */ + block->is_reachable = true; + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +bool +aot_compile_op_end(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block; + LLVMValueRef value; + LLVMBasicBlockRef next_llvm_end_block; + char name[32]; + uint32 i, result_index; + + /* Check block stack */ + if (!(block = func_ctx->block_stack.block_list_end)) { + aot_set_last_error("WASM block stack underflow."); + return false; + } + + /* Create the end block */ + if (!block->llvm_end_block) { + format_block_name(name, sizeof(name), block->block_index, + block->label_type, LABEL_END); + CREATE_BLOCK(block->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block))) + MOVE_BLOCK_BEFORE(block->llvm_end_block, next_llvm_end_block); + } + + /* Handle block result values */ + CREATE_RESULT_VALUE_PHIS(block); + for (i = 0; i < block->result_count; i++) { + value = NULL; + result_index = block->result_count - 1 - i; + POP(value, block->result_types[result_index]); + bh_assert(value); + ADD_TO_RESULT_PHIS(block, value, result_index); + } + + /* Jump to the end block */ + BUILD_BR(block->llvm_end_block); + + block->is_reachable = true; + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +#if WASM_ENABLE_THREAD_MGR != 0 +bool +check_suspend_flags(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef terminate_addr, terminate_flags, flag, offset, res; + LLVMBasicBlockRef terminate_block, non_terminate_block; + AOTFuncType *aot_func_type = func_ctx->aot_func->func_type; + bool is_shared_memory = + comp_ctx->comp_data->memories[0].memory_flags & 0x02 ? true : false; + + /* Only need to check the suspend flags when memory is shared since + shared memory must be enabled for multi-threading */ + if (!is_shared_memory) { + return true; + } + + /* Offset of suspend_flags */ + offset = I32_FIVE; + + if (!(terminate_addr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, OPQ_PTR_TYPE, func_ctx->exec_env, &offset, 1, + "terminate_addr"))) { + aot_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(terminate_addr = + LLVMBuildBitCast(comp_ctx->builder, terminate_addr, + INT32_PTR_TYPE, "terminate_addr_ptr"))) { + aot_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(terminate_flags = + LLVMBuildLoad2(comp_ctx->builder, I32_TYPE, terminate_addr, + "terminate_flags"))) { + aot_set_last_error("llvm build LOAD failed"); + return false; + } + /* Set terminate_flags memory accecc to volatile, so that the value + will always be loaded from memory rather than register */ + LLVMSetVolatile(terminate_flags, true); + + if (!(flag = LLVMBuildAnd(comp_ctx->builder, terminate_flags, I32_ONE, + "termination_flag"))) { + aot_set_last_error("llvm build AND failed"); + return false; + } + + CREATE_BLOCK(non_terminate_block, "non_terminate"); + MOVE_BLOCK_AFTER_CURR(non_terminate_block); + + CREATE_BLOCK(terminate_block, "terminate"); + MOVE_BLOCK_AFTER_CURR(terminate_block); + + BUILD_ICMP(LLVMIntEQ, flag, I32_ZERO, res, "flag_terminate"); + BUILD_COND_BR(res, non_terminate_block, terminate_block); + + /* Move builder to terminate block */ + SET_BUILDER_POS(terminate_block); + if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + goto fail; + } + + /* Move builder to non terminate block */ + SET_BUILDER_POS(non_terminate_block); + return true; + +fail: + return false; +} +#endif /* End of WASM_ENABLE_THREAD_MGR */ + +bool +aot_compile_op_br(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip) +{ + AOTBlock *block_dst; + LLVMValueRef value_ret, value_param; + LLVMBasicBlockRef next_llvm_end_block; + char name[32]; + uint32 i, param_index, result_index; + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + + if (!(block_dst = get_target_block(func_ctx, br_depth))) { + return false; + } + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + /* Dest block is Loop block */ + /* Handle Loop parameters */ + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value_param, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value_param, param_index); + } + BUILD_BR(block_dst->llvm_entry_block); + } + else { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!block_dst->llvm_end_block) { + format_block_name(name, sizeof(name), block_dst->block_index, + block_dst->label_type, LABEL_END); + CREATE_BLOCK(block_dst->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) + MOVE_BLOCK_BEFORE(block_dst->llvm_end_block, + next_llvm_end_block); + } + + block_dst->is_reachable = true; + + /* Handle result values */ + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value_ret, block_dst->result_types[result_index]); + ADD_TO_RESULT_PHIS(block_dst, value_ret, result_index); + } + /* Jump to the end block */ + BUILD_BR(block_dst->llvm_end_block); + } + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +bool +aot_compile_op_br_if(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 br_depth, uint8 **p_frame_ip) +{ + AOTBlock *block_dst; + LLVMValueRef value_cmp, value, *values = NULL; + LLVMBasicBlockRef llvm_else_block, next_llvm_end_block; + char name[32]; + uint32 i, param_index, result_index; + uint64 size; + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + + POP_COND(value_cmp); + + if (LLVMIsUndef(value_cmp) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(value_cmp) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + + if (!LLVMIsConstant(value_cmp)) { + /* Compare value is not constant, create condition br IR */ + if (!(block_dst = get_target_block(func_ctx, br_depth))) { + return false; + } + + /* Create llvm else block */ + CREATE_BLOCK(llvm_else_block, "br_if_else"); + MOVE_BLOCK_AFTER_CURR(llvm_else_block); + + if (block_dst->label_type == LABEL_TYPE_LOOP) { + /* Dest block is Loop block */ + /* Handle Loop parameters */ + if (block_dst->param_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (i = 0; i < block_dst->param_count; i++) { + param_index = block_dst->param_count - 1 - i; + POP(value, block_dst->param_types[param_index]); + ADD_TO_PARAM_PHIS(block_dst, value, param_index); + values[param_index] = value; + } + for (i = 0; i < block_dst->param_count; i++) { + PUSH(values[i], block_dst->param_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + + BUILD_COND_BR(value_cmp, block_dst->llvm_entry_block, + llvm_else_block); + + /* Move builder to else block */ + SET_BUILDER_POS(llvm_else_block); + } + else { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!block_dst->llvm_end_block) { + format_block_name(name, sizeof(name), block_dst->block_index, + block_dst->label_type, LABEL_END); + CREATE_BLOCK(block_dst->llvm_end_block, name); + if ((next_llvm_end_block = find_next_llvm_end_block(block_dst))) + MOVE_BLOCK_BEFORE(block_dst->llvm_end_block, + next_llvm_end_block); + } + + /* Set reachable flag and create condition br IR */ + block_dst->is_reachable = true; + + /* Handle result values */ + if (block_dst->result_count) { + size = sizeof(LLVMValueRef) * (uint64)block_dst->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(block_dst); + for (i = 0; i < block_dst->result_count; i++) { + result_index = block_dst->result_count - 1 - i; + POP(value, block_dst->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(block_dst, value, result_index); + } + for (i = 0; i < block_dst->result_count; i++) { + PUSH(values[i], block_dst->result_types[i]); + } + wasm_runtime_free(values); + values = NULL; + } + + /* Condition jump to end block */ + BUILD_COND_BR(value_cmp, block_dst->llvm_end_block, + llvm_else_block); + + /* Move builder to else block */ + SET_BUILDER_POS(llvm_else_block); + } + } + else { + if ((int32)LLVMConstIntGetZExtValue(value_cmp) != 0) { + /* Compare value is not 0, condition is true, same as op_br */ + return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip); + } + else { + /* Compare value is not 0, condition is false, skip br_if */ + return true; + } + } + return true; +fail: + if (values) + wasm_runtime_free(values); + return false; +} + +bool +aot_compile_op_br_table(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint32 *br_depths, uint32 br_count, uint8 **p_frame_ip) +{ + uint32 i, j; + LLVMValueRef value_switch, value_cmp, value_case, value, *values = NULL; + LLVMBasicBlockRef default_llvm_block = NULL, target_llvm_block; + LLVMBasicBlockRef next_llvm_end_block; + AOTBlock *target_block; + uint32 br_depth, depth_idx; + uint32 param_index, result_index; + uint64 size; + char name[32]; + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + return false; + } +#endif + + POP_I32(value_cmp); + + if (LLVMIsUndef(value_cmp) +#if LLVM_VERSION_NUMBER >= 12 + || LLVMIsPoison(value_cmp) +#endif + ) { + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_INTEGER_OVERFLOW, + false, NULL, NULL))) { + goto fail; + } + return aot_handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + + if (!LLVMIsConstant(value_cmp)) { + /* Compare value is not constant, create switch IR */ + for (i = 0; i <= br_count; i++) { + target_block = get_target_block(func_ctx, br_depths[i]); + if (!target_block) + return false; + + if (target_block->label_type != LABEL_TYPE_LOOP) { + /* Dest block is Block/If/Function block */ + /* Create the end block */ + if (!target_block->llvm_end_block) { + format_block_name(name, sizeof(name), + target_block->block_index, + target_block->label_type, LABEL_END); + CREATE_BLOCK(target_block->llvm_end_block, name); + if ((next_llvm_end_block = + find_next_llvm_end_block(target_block))) + MOVE_BLOCK_BEFORE(target_block->llvm_end_block, + next_llvm_end_block); + } + /* Handle result values */ + if (target_block->result_count) { + size = sizeof(LLVMValueRef) + * (uint64)target_block->result_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + CREATE_RESULT_VALUE_PHIS(target_block); + for (j = 0; j < target_block->result_count; j++) { + result_index = target_block->result_count - 1 - j; + POP(value, target_block->result_types[result_index]); + values[result_index] = value; + ADD_TO_RESULT_PHIS(target_block, value, result_index); + } + for (j = 0; j < target_block->result_count; j++) { + PUSH(values[j], target_block->result_types[j]); + } + wasm_runtime_free(values); + } + target_block->is_reachable = true; + if (i == br_count) + default_llvm_block = target_block->llvm_end_block; + } + else { + /* Handle Loop parameters */ + if (target_block->param_count) { + size = sizeof(LLVMValueRef) + * (uint64)target_block->param_count; + if (size >= UINT32_MAX + || !(values = wasm_runtime_malloc((uint32)size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + for (j = 0; j < target_block->param_count; j++) { + param_index = target_block->param_count - 1 - j; + POP(value, target_block->param_types[param_index]); + values[param_index] = value; + ADD_TO_PARAM_PHIS(target_block, value, param_index); + } + for (j = 0; j < target_block->param_count; j++) { + PUSH(values[j], target_block->param_types[j]); + } + wasm_runtime_free(values); + } + if (i == br_count) + default_llvm_block = target_block->llvm_entry_block; + } + } + + /* Create switch IR */ + if (!(value_switch = LLVMBuildSwitch(comp_ctx->builder, value_cmp, + default_llvm_block, br_count))) { + aot_set_last_error("llvm build switch failed."); + return false; + } + + /* Add each case for switch IR */ + for (i = 0; i < br_count; i++) { + value_case = I32_CONST(i); + CHECK_LLVM_CONST(value_case); + target_block = get_target_block(func_ctx, br_depths[i]); + if (!target_block) + return false; + target_llvm_block = target_block->label_type != LABEL_TYPE_LOOP + ? target_block->llvm_end_block + : target_block->llvm_entry_block; + LLVMAddCase(value_switch, value_case, target_llvm_block); + } + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); + } + else { + /* Compare value is constant, create br IR */ + depth_idx = (uint32)LLVMConstIntGetZExtValue(value_cmp); + br_depth = br_depths[br_count]; + if (depth_idx < br_count) { + br_depth = br_depths[depth_idx]; + } + return aot_compile_op_br(comp_ctx, func_ctx, br_depth, p_frame_ip); + } +fail: + if (values) + wasm_runtime_free(values); + return false; +} + +bool +aot_compile_op_return(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + AOTBlock *block_func = func_ctx->block_stack.block_list_head; + LLVMValueRef value; + LLVMValueRef ret; + AOTFuncType *func_type; + uint32 i, param_index, result_index; +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMMetadataRef return_location; +#endif + + bh_assert(block_func); + func_type = func_ctx->aot_func->func_type; + +#if WASM_ENABLE_DEBUG_AOT != 0 + return_location = dwarf_gen_location( + comp_ctx, func_ctx, + (*p_frame_ip - 1) - comp_ctx->comp_data->wasm_module->buf_code); +#endif + if (block_func->result_count) { + /* Store extra result values to function parameters */ + for (i = 0; i < block_func->result_count - 1; i++) { + result_index = block_func->result_count - 1 - i; + POP(value, block_func->result_types[result_index]); + param_index = func_type->param_count + result_index; + if (!LLVMBuildStore(comp_ctx->builder, value, + LLVMGetParam(func_ctx->func, param_index))) { + aot_set_last_error("llvm build store failed."); + goto fail; + } + } + /* Return the first result value */ + POP(value, block_func->result_types[0]); + if (!(ret = LLVMBuildRet(comp_ctx->builder, value))) { + aot_set_last_error("llvm build return failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMInstructionSetDebugLoc(ret, return_location); +#endif + } + else { + if (!(ret = LLVMBuildRetVoid(comp_ctx->builder))) { + aot_set_last_error("llvm build return void failed."); + goto fail; + } +#if WASM_ENABLE_DEBUG_AOT != 0 + LLVMInstructionSetDebugLoc(ret, return_location); +#endif + } + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +fail: + return false; +} + +bool +aot_compile_op_unreachable(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + uint8 **p_frame_ip) +{ + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_UNREACHABLE, false, NULL, + NULL)) + return false; + + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +} + +bool +aot_handle_next_reachable_block(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, uint8 **p_frame_ip) +{ + return handle_next_reachable_block(comp_ctx, func_ctx, p_frame_ip); +} |