From c21c3b0befeb46a51b6bf3758ffa30813bea0ff0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 9 Mar 2024 14:19:22 +0100 Subject: Adding upstream version 1.44.3. Signed-off-by: Daniel Baumann --- .../core/iwasm/fast-jit/jit_frontend.c | 2387 ++++++++++++++++++++ 1 file changed, 2387 insertions(+) create mode 100644 fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/fast-jit/jit_frontend.c (limited to 'fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/fast-jit/jit_frontend.c') diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/fast-jit/jit_frontend.c b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/fast-jit/jit_frontend.c new file mode 100644 index 000000000..ec68ad91d --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/core/iwasm/fast-jit/jit_frontend.c @@ -0,0 +1,2387 @@ +/* + * Copyright (C) 2021 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include "jit_compiler.h" +#include "jit_frontend.h" +#include "fe/jit_emit_compare.h" +#include "fe/jit_emit_const.h" +#include "fe/jit_emit_control.h" +#include "fe/jit_emit_conversion.h" +#include "fe/jit_emit_exception.h" +#include "fe/jit_emit_function.h" +#include "fe/jit_emit_memory.h" +#include "fe/jit_emit_numberic.h" +#include "fe/jit_emit_parametric.h" +#include "fe/jit_emit_table.h" +#include "fe/jit_emit_variable.h" +#include "../interpreter/wasm_interp.h" +#include "../interpreter/wasm_opcode.h" +#include "../interpreter/wasm_runtime.h" +#include "../common/wasm_exec_env.h" + +static uint32 +get_global_base_offset(const WASMModule *module) +{ + uint32 module_inst_struct_size = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes); + uint32 mem_inst_size = + (uint32)sizeof(WASMMemoryInstance) + * (module->import_memory_count + module->memory_count); + +#if WASM_ENABLE_JIT != 0 + /* If the module dosen't have memory, reserve one mem_info space + with empty content to align with llvm jit compiler */ + if (mem_inst_size == 0) + mem_inst_size = (uint32)sizeof(WASMMemoryInstance); +#endif + + /* Size of module inst and memory instances */ + return module_inst_struct_size + mem_inst_size; +} + +static uint32 +get_first_table_inst_offset(const WASMModule *module) +{ + return get_global_base_offset(module) + module->global_data_size; +} + +uint32 +jit_frontend_get_global_data_offset(const WASMModule *module, uint32 global_idx) +{ + uint32 global_base_offset = get_global_base_offset(module); + + if (global_idx < module->import_global_count) { + const WASMGlobalImport *import_global = + &((module->import_globals + global_idx)->u.global); + return global_base_offset + import_global->data_offset; + } + else { + const WASMGlobal *global = + module->globals + (global_idx - module->import_global_count); + return global_base_offset + global->data_offset; + } +} + +uint32 +jit_frontend_get_table_inst_offset(const WASMModule *module, uint32 tbl_idx) +{ + uint32 offset, i = 0; + + offset = get_first_table_inst_offset(module); + + while (i < tbl_idx && i < module->import_table_count) { + WASMTableImport *import_table = &module->import_tables[i].u.table; + + offset += (uint32)offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + offset += (uint32)sizeof(uint32) * import_table->max_size; +#else + offset += (uint32)sizeof(uint32) + * (import_table->possible_grow ? import_table->max_size + : import_table->init_size); +#endif + + i++; + } + + if (i == tbl_idx) { + return offset; + } + + tbl_idx -= module->import_table_count; + i -= module->import_table_count; + while (i < tbl_idx && i < module->table_count) { + WASMTable *table = module->tables + i; + + offset += (uint32)offsetof(WASMTableInstance, elems); +#if WASM_ENABLE_MULTI_MODULE != 0 + offset += (uint32)sizeof(uint32) * table->max_size; +#else + offset += (uint32)sizeof(uint32) + * (table->possible_grow ? table->max_size : table->init_size); +#endif + + i++; + } + + return offset; +} + +uint32 +jit_frontend_get_module_inst_extra_offset(const WASMModule *module) +{ + uint32 offset = jit_frontend_get_table_inst_offset( + module, module->import_table_count + module->table_count); + + return align_uint(offset, 8); +} + +JitReg +get_module_inst_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + + if (!frame->module_inst_reg) { + frame->module_inst_reg = cc->module_inst_reg; + GEN_INSN(LDPTR, frame->module_inst_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); + } + return frame->module_inst_reg; +} + +JitReg +get_module_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->module_reg) { + frame->module_reg = cc->module_reg; + GEN_INSN(LDPTR, frame->module_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, module))); + } + return frame->module_reg; +} + +JitReg +get_import_func_ptrs_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->import_func_ptrs_reg) { + frame->import_func_ptrs_reg = cc->import_func_ptrs_reg; + GEN_INSN( + LDPTR, frame->import_func_ptrs_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, import_func_ptrs))); + } + return frame->import_func_ptrs_reg; +} + +JitReg +get_fast_jit_func_ptrs_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->fast_jit_func_ptrs_reg) { + frame->fast_jit_func_ptrs_reg = cc->fast_jit_func_ptrs_reg; + GEN_INSN( + LDPTR, frame->fast_jit_func_ptrs_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, fast_jit_func_ptrs))); + } + return frame->fast_jit_func_ptrs_reg; +} + +JitReg +get_func_type_indexes_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + + if (!frame->func_type_indexes_reg) { + frame->func_type_indexes_reg = cc->func_type_indexes_reg; + GEN_INSN( + LDPTR, frame->func_type_indexes_reg, module_inst_reg, + NEW_CONST(I32, offsetof(WASMModuleInstance, func_type_indexes))); + } + return frame->func_type_indexes_reg; +} + +JitReg +get_aux_stack_bound_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + + if (!frame->aux_stack_bound_reg) { + frame->aux_stack_bound_reg = cc->aux_stack_bound_reg; + GEN_INSN( + LDI32, frame->aux_stack_bound_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_boundary.boundary))); + } + return frame->aux_stack_bound_reg; +} + +JitReg +get_aux_stack_bottom_reg(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + + if (!frame->aux_stack_bottom_reg) { + frame->aux_stack_bottom_reg = cc->aux_stack_bottom_reg; + GEN_INSN( + LDI32, frame->aux_stack_bottom_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, aux_stack_bottom.bottom))); + } + return frame->aux_stack_bottom_reg; +} + +JitReg +get_memory_data_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 memory_data_offset; + + bh_assert(mem_idx == 0); +#if WASM_ENABLE_SHARED_MEMORY != 0 + uint32 memories_offset = (uint32)offsetof(WASMModuleInstance, memories); + JitReg memories_addr = jit_cc_new_reg_ptr(cc); + JitReg memories_0_addr = jit_cc_new_reg_ptr(cc); + memory_data_offset = (uint32)offsetof(WASMMemoryInstance, memory_data); + if (!frame->memory_regs[mem_idx].memory_data) { + frame->memory_regs[mem_idx].memory_data = + cc->memory_regs[mem_idx].memory_data; + /* module_inst->memories */ + GEN_INSN(LDPTR, memories_addr, module_inst_reg, + NEW_CONST(I32, memories_offset)); + /* module_inst->memories[0] */ + GEN_INSN(LDPTR, memories_0_addr, memories_addr, NEW_CONST(I32, 0)); + /* memories[0]->memory_data */ + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data, + memories_0_addr, NEW_CONST(I32, memory_data_offset)); + } +#else + memory_data_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, memory_data); + if (!frame->memory_regs[mem_idx].memory_data) { + frame->memory_regs[mem_idx].memory_data = + cc->memory_regs[mem_idx].memory_data; + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data, + module_inst_reg, NEW_CONST(I32, memory_data_offset)); + } +#endif + return frame->memory_regs[mem_idx].memory_data; +} + +JitReg +get_memory_data_end_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 memory_data_end_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, memory_data_end); + + bh_assert(mem_idx == 0); + + if (!frame->memory_regs[mem_idx].memory_data_end) { + frame->memory_regs[mem_idx].memory_data_end = + cc->memory_regs[mem_idx].memory_data_end; + GEN_INSN(LDPTR, frame->memory_regs[mem_idx].memory_data_end, + module_inst_reg, NEW_CONST(I32, memory_data_end_offset)); + } + return frame->memory_regs[mem_idx].memory_data_end; +} + +JitReg +get_mem_bound_check_1byte_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 mem_bound_check_1byte_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_1byte); + + bh_assert(mem_idx == 0); + + if (!frame->memory_regs[mem_idx].mem_bound_check_1byte) { + frame->memory_regs[mem_idx].mem_bound_check_1byte = + cc->memory_regs[mem_idx].mem_bound_check_1byte; +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_1byte, + module_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_1byte, + module_inst_reg, NEW_CONST(I32, mem_bound_check_1byte_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_1byte; +} + +JitReg +get_mem_bound_check_2bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 mem_bound_check_2bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_2bytes); + + bh_assert(mem_idx == 0); + + if (!frame->memory_regs[mem_idx].mem_bound_check_2bytes) { + frame->memory_regs[mem_idx].mem_bound_check_2bytes = + cc->memory_regs[mem_idx].mem_bound_check_2bytes; +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_2bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_2bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_2bytes; +} + +JitReg +get_mem_bound_check_4bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 mem_bound_check_4bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_4bytes); + + bh_assert(mem_idx == 0); + + if (!frame->memory_regs[mem_idx].mem_bound_check_4bytes) { + frame->memory_regs[mem_idx].mem_bound_check_4bytes = + cc->memory_regs[mem_idx].mem_bound_check_4bytes; +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_4bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_4bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_4bytes; +} + +JitReg +get_mem_bound_check_8bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 mem_bound_check_8bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_8bytes); + + bh_assert(mem_idx == 0); + + if (!frame->memory_regs[mem_idx].mem_bound_check_8bytes) { + frame->memory_regs[mem_idx].mem_bound_check_8bytes = + cc->memory_regs[mem_idx].mem_bound_check_8bytes; +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_8bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_8bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_8bytes; +} + +JitReg +get_mem_bound_check_16bytes_reg(JitFrame *frame, uint32 mem_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst_reg = get_module_inst_reg(frame); + uint32 mem_bound_check_16bytes_offset = + (uint32)offsetof(WASMModuleInstance, global_table_data.bytes) + + (uint32)offsetof(WASMMemoryInstance, mem_bound_check_16bytes); + + bh_assert(mem_idx == 0); + + if (!frame->memory_regs[mem_idx].mem_bound_check_16bytes) { + frame->memory_regs[mem_idx].mem_bound_check_16bytes = + cc->memory_regs[mem_idx].mem_bound_check_16bytes; +#if UINTPTR_MAX == UINT64_MAX + GEN_INSN(LDI64, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#else + GEN_INSN(LDI32, frame->memory_regs[mem_idx].mem_bound_check_16bytes, + module_inst_reg, + NEW_CONST(I32, mem_bound_check_16bytes_offset)); +#endif + } + return frame->memory_regs[mem_idx].mem_bound_check_16bytes; +} + +JitReg +get_table_elems_reg(JitFrame *frame, uint32 tbl_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst = get_module_inst_reg(frame); + uint32 offset = + jit_frontend_get_table_inst_offset(cc->cur_wasm_module, tbl_idx) + + (uint32)offsetof(WASMTableInstance, elems); + + if (!frame->table_regs[tbl_idx].table_elems) { + frame->table_regs[tbl_idx].table_elems = + cc->table_regs[tbl_idx].table_elems; + GEN_INSN(ADD, frame->table_regs[tbl_idx].table_elems, module_inst, + NEW_CONST(PTR, offset)); + } + return frame->table_regs[tbl_idx].table_elems; +} + +JitReg +get_table_cur_size_reg(JitFrame *frame, uint32 tbl_idx) +{ + JitCompContext *cc = frame->cc; + JitReg module_inst = get_module_inst_reg(frame); + uint32 offset = + jit_frontend_get_table_inst_offset(cc->cur_wasm_module, tbl_idx) + + (uint32)offsetof(WASMTableInstance, cur_size); + + if (!frame->table_regs[tbl_idx].table_cur_size) { + frame->table_regs[tbl_idx].table_cur_size = + cc->table_regs[tbl_idx].table_cur_size; + GEN_INSN(LDI32, frame->table_regs[tbl_idx].table_cur_size, module_inst, + NEW_CONST(I32, offset)); + } + return frame->table_regs[tbl_idx].table_cur_size; +} + +void +clear_fixed_virtual_regs(JitFrame *frame) +{ + WASMModule *module = frame->cc->cur_wasm_module; + uint32 count, i; + + frame->module_inst_reg = 0; + frame->module_reg = 0; + frame->import_func_ptrs_reg = 0; + frame->fast_jit_func_ptrs_reg = 0; + frame->func_type_indexes_reg = 0; + frame->aux_stack_bound_reg = 0; + frame->aux_stack_bottom_reg = 0; + + count = module->import_memory_count + module->memory_count; + for (i = 0; i < count; i++) { + frame->memory_regs[i].memory_data = 0; + frame->memory_regs[i].memory_data_end = 0; + frame->memory_regs[i].mem_bound_check_1byte = 0; + frame->memory_regs[i].mem_bound_check_2bytes = 0; + frame->memory_regs[i].mem_bound_check_4bytes = 0; + frame->memory_regs[i].mem_bound_check_8bytes = 0; + frame->memory_regs[i].mem_bound_check_16bytes = 0; + } + + count = module->import_table_count + module->table_count; + for (i = 0; i < count; i++) { + frame->table_regs[i].table_elems = 0; + frame->table_regs[i].table_cur_size = 0; + } +} + +void +clear_memory_regs(JitFrame *frame) +{ + WASMModule *module = frame->cc->cur_wasm_module; + uint32 count, i; + + count = module->import_memory_count + module->memory_count; + for (i = 0; i < count; i++) { + frame->memory_regs[i].memory_data = 0; + frame->memory_regs[i].memory_data_end = 0; + frame->memory_regs[i].mem_bound_check_1byte = 0; + frame->memory_regs[i].mem_bound_check_2bytes = 0; + frame->memory_regs[i].mem_bound_check_4bytes = 0; + frame->memory_regs[i].mem_bound_check_8bytes = 0; + frame->memory_regs[i].mem_bound_check_16bytes = 0; + } +} + +void +clear_table_regs(JitFrame *frame) +{ + WASMModule *module = frame->cc->cur_wasm_module; + uint32 count, i; + + count = module->import_table_count + module->table_count; + for (i = 0; i < count; i++) { + frame->table_regs[i].table_cur_size = 0; + } +} + +JitReg +gen_load_i32(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = jit_cc_new_reg_I32(cc); + GEN_INSN(LDI32, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +JitReg +gen_load_i64(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = frame->lp[n + 1].reg = jit_cc_new_reg_I64(cc); + GEN_INSN(LDI64, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +JitReg +gen_load_f32(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = jit_cc_new_reg_F32(cc); + GEN_INSN(LDF32, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +JitReg +gen_load_f64(JitFrame *frame, unsigned n) +{ + if (!frame->lp[n].reg) { + JitCompContext *cc = frame->cc; + frame->lp[n].reg = frame->lp[n + 1].reg = jit_cc_new_reg_F64(cc); + GEN_INSN(LDF64, frame->lp[n].reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + } + + return frame->lp[n].reg; +} + +void +gen_commit_values(JitFrame *frame, JitValueSlot *begin, JitValueSlot *end) +{ + JitCompContext *cc = frame->cc; + JitValueSlot *p; + int n; + + for (p = begin; p < end; p++) { + if (!p->dirty) + continue; + + p->dirty = 0; + n = p - frame->lp; + + switch (jit_reg_kind(p->reg)) { + case JIT_REG_KIND_I32: + GEN_INSN(STI32, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + + case JIT_REG_KIND_I64: + GEN_INSN(STI64, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + (++p)->dirty = 0; + break; + + case JIT_REG_KIND_F32: + GEN_INSN(STF32, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + break; + + case JIT_REG_KIND_F64: + GEN_INSN(STF64, p->reg, cc->fp_reg, + NEW_CONST(I32, offset_of_local(n))); + (++p)->dirty = 0; + break; + } + } +} + +/** + * Generate instructions to commit SP and IP pointers to the frame. + * + * @param frame the frame information + */ +void +gen_commit_sp_ip(JitFrame *frame) +{ + JitCompContext *cc = frame->cc; + JitReg sp; + + if (frame->sp != frame->committed_sp) { + sp = jit_cc_new_reg_ptr(cc); + GEN_INSN(ADD, sp, cc->fp_reg, + NEW_CONST(PTR, offset_of_local(frame->sp - frame->lp))); + GEN_INSN(STPTR, sp, cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + frame->committed_sp = frame->sp; + } + +#if 0 /* Disable committing ip currently */ + if (frame->ip != frame->committed_ip) { + GEN_INSN(STPTR, NEW_CONST(PTR, (uintptr_t)frame->ip), cc->fp_reg, + NEW_CONST(I32, offsetof(WASMInterpFrame, ip))); + frame->committed_ip = frame->ip; + } +#endif +} + +static bool +create_fixed_virtual_regs(JitCompContext *cc) +{ + WASMModule *module = cc->cur_wasm_module; + uint64 total_size; + uint32 i, count; + + cc->module_inst_reg = jit_cc_new_reg_ptr(cc); + cc->module_reg = jit_cc_new_reg_ptr(cc); + cc->import_func_ptrs_reg = jit_cc_new_reg_ptr(cc); + cc->fast_jit_func_ptrs_reg = jit_cc_new_reg_ptr(cc); + cc->func_type_indexes_reg = jit_cc_new_reg_ptr(cc); + cc->aux_stack_bound_reg = jit_cc_new_reg_I32(cc); + cc->aux_stack_bottom_reg = jit_cc_new_reg_I32(cc); + + count = module->import_memory_count + module->memory_count; + if (count > 0) { + total_size = (uint64)sizeof(JitMemRegs) * count; + if (total_size > UINT32_MAX + || !(cc->memory_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + for (i = 0; i < count; i++) { + cc->memory_regs[i].memory_data = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].memory_data_end = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_1byte = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_2bytes = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_4bytes = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_8bytes = jit_cc_new_reg_ptr(cc); + cc->memory_regs[i].mem_bound_check_16bytes = jit_cc_new_reg_ptr(cc); + } + } + + count = module->import_table_count + module->table_count; + if (count > 0) { + total_size = (uint64)sizeof(JitTableRegs) * count; + if (total_size > UINT32_MAX + || !(cc->table_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + return false; + } + + for (i = 0; i < count; i++) { + cc->table_regs[i].table_elems = jit_cc_new_reg_ptr(cc); + cc->table_regs[i].table_cur_size = jit_cc_new_reg_I32(cc); + } + } + + return true; +} + +static bool +form_and_translate_func(JitCompContext *cc) +{ + JitBasicBlock *func_entry_basic_block; + JitReg func_entry_label; + JitInsn *insn; + JitIncomingInsn *incoming_insn, *incoming_insn_next; + uint32 i; + + if (!create_fixed_virtual_regs(cc)) + return false; + + if (!(func_entry_basic_block = jit_frontend_translate_func(cc))) + return false; + + jit_cc_reset_insn_hash(cc); + + /* The label of the func entry basic block. */ + func_entry_label = jit_basic_block_label(func_entry_basic_block); + + /* Create a JMP instruction jumping to the func entry. */ + if (!(insn = jit_cc_new_insn(cc, JMP, func_entry_label))) + return false; + + /* Insert the instruction into the cc entry block. */ + jit_basic_block_append_insn(jit_cc_entry_basic_block(cc), insn); + + /* Patch INSNs jumping to exception basic blocks. */ + for (i = 0; i < EXCE_NUM; i++) { + incoming_insn = cc->incoming_insns_for_exec_bbs[i]; + if (incoming_insn) { + if (!(cc->exce_basic_blocks[i] = jit_cc_new_basic_block(cc, 0))) { + jit_set_last_error(cc, "create basic block failed"); + return false; + } + while (incoming_insn) { + incoming_insn_next = incoming_insn->next; + insn = incoming_insn->insn; + if (insn->opcode == JIT_OP_JMP) { + *(jit_insn_opnd(insn, 0)) = + jit_basic_block_label(cc->exce_basic_blocks[i]); + } + else if (insn->opcode >= JIT_OP_BEQ + && insn->opcode <= JIT_OP_BLEU) { + *(jit_insn_opnd(insn, 1)) = + jit_basic_block_label(cc->exce_basic_blocks[i]); + } + incoming_insn = incoming_insn_next; + } + cc->cur_basic_block = cc->exce_basic_blocks[i]; + if (i != EXCE_ALREADY_THROWN) { + JitReg module_inst_reg = jit_cc_new_reg_ptr(cc); + GEN_INSN(LDPTR, module_inst_reg, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); + insn = GEN_INSN( + CALLNATIVE, 0, + NEW_CONST(PTR, (uintptr_t)jit_set_exception_with_id), 2); + if (insn) { + *(jit_insn_opndv(insn, 2)) = module_inst_reg; + *(jit_insn_opndv(insn, 3)) = NEW_CONST(I32, i); + } + } + GEN_INSN(RETURN, NEW_CONST(I32, JIT_INTERP_ACTION_THROWN)); + + *(jit_annl_begin_bcip(cc, + jit_basic_block_label(cc->cur_basic_block))) = + *(jit_annl_end_bcip( + cc, jit_basic_block_label(cc->cur_basic_block))) = + cc->cur_wasm_module->load_addr; + } + } + + *(jit_annl_begin_bcip(cc, cc->entry_label)) = + *(jit_annl_end_bcip(cc, cc->entry_label)) = + *(jit_annl_begin_bcip(cc, cc->exit_label)) = + *(jit_annl_end_bcip(cc, cc->exit_label)) = + cc->cur_wasm_module->load_addr; + + if (jit_get_last_error(cc)) { + return false; + } + return true; +} + +bool +jit_pass_frontend(JitCompContext *cc) +{ + /* Enable necessary annotations required at the current stage. */ + if (!jit_annl_enable_begin_bcip(cc) || !jit_annl_enable_end_bcip(cc) + || !jit_annl_enable_end_sp(cc) || !jit_annr_enable_def_insn(cc) + || !jit_cc_enable_insn_hash(cc, 127)) + return false; + + if (!(form_and_translate_func(cc))) + return false; + + /* Release the annotations after local CSE and translation. */ + jit_cc_disable_insn_hash(cc); + jit_annl_disable_end_sp(cc); + + return true; +} + +static JitFrame * +init_func_translation(JitCompContext *cc) +{ + JitFrame *jit_frame; + JitReg top, top_boundary, new_top, frame_boundary, frame_sp; + WASMModule *cur_wasm_module = cc->cur_wasm_module; + WASMFunction *cur_wasm_func = cc->cur_wasm_func; + uint32 cur_wasm_func_idx = cc->cur_wasm_func_idx; + uint32 max_locals = + cur_wasm_func->param_cell_num + cur_wasm_func->local_cell_num; + uint32 max_stacks = cur_wasm_func->max_stack_cell_num; + uint64 total_cell_num = + (uint64)cur_wasm_func->param_cell_num + + (uint64)cur_wasm_func->local_cell_num + + (uint64)cur_wasm_func->max_stack_cell_num + + ((uint64)cur_wasm_func->max_block_num) * sizeof(WASMBranchBlock) / 4; + uint32 frame_size, outs_size, local_size, count; + uint32 i, local_off; + uint64 total_size; +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 + JitReg module_inst, func_inst; + uint32 func_insts_offset; +#if WASM_ENABLE_PERF_PROFILING != 0 + JitReg time_started; +#endif +#endif + + if ((uint64)max_locals + (uint64)max_stacks >= UINT32_MAX + || total_cell_num >= UINT32_MAX + || !(jit_frame = jit_calloc(offsetof(JitFrame, lp) + + sizeof(*jit_frame->lp) + * (max_locals + max_stacks)))) { + os_printf("allocate jit frame failed\n"); + return NULL; + } + + count = + cur_wasm_module->import_memory_count + cur_wasm_module->memory_count; + if (count > 0) { + total_size = (uint64)sizeof(JitMemRegs) * count; + if (total_size > UINT32_MAX + || !(jit_frame->memory_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + jit_free(jit_frame); + return NULL; + } + } + + count = cur_wasm_module->import_table_count + cur_wasm_module->table_count; + if (count > 0) { + total_size = (uint64)sizeof(JitTableRegs) * count; + if (total_size > UINT32_MAX + || !(jit_frame->table_regs = jit_calloc((uint32)total_size))) { + jit_set_last_error(cc, "allocate memory failed"); + if (jit_frame->memory_regs) + jit_free(jit_frame->memory_regs); + jit_free(jit_frame); + return NULL; + } + } + + jit_frame->cur_wasm_module = cur_wasm_module; + jit_frame->cur_wasm_func = cur_wasm_func; + jit_frame->cur_wasm_func_idx = cur_wasm_func_idx; + jit_frame->cc = cc; + jit_frame->max_locals = max_locals; + jit_frame->max_stacks = max_stacks; + jit_frame->sp = jit_frame->lp + max_locals; + jit_frame->ip = cur_wasm_func->code; + + cc->jit_frame = jit_frame; + cc->cur_basic_block = jit_cc_entry_basic_block(cc); + cc->spill_cache_offset = wasm_interp_interp_frame_size(total_cell_num); + /* Set spill cache size according to max local cell num, max stack cell + num and virtual fixed register num */ + cc->spill_cache_size = (max_locals + max_stacks) * 4 + sizeof(void *) * 16; + cc->total_frame_size = cc->spill_cache_offset + cc->spill_cache_size; + cc->jitted_return_address_offset = + offsetof(WASMInterpFrame, jitted_return_addr); + cc->cur_basic_block = jit_cc_entry_basic_block(cc); + + frame_size = outs_size = cc->total_frame_size; + local_size = + (cur_wasm_func->param_cell_num + cur_wasm_func->local_cell_num) * 4; + + top = jit_cc_new_reg_ptr(cc); + top_boundary = jit_cc_new_reg_ptr(cc); + new_top = jit_cc_new_reg_ptr(cc); + frame_boundary = jit_cc_new_reg_ptr(cc); + frame_sp = jit_cc_new_reg_ptr(cc); + +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 + module_inst = jit_cc_new_reg_ptr(cc); + func_inst = jit_cc_new_reg_ptr(cc); +#if WASM_ENABLE_PERF_PROFILING != 0 + time_started = jit_cc_new_reg_I64(cc); + /* Call os_time_get_boot_microsecond() to get time_started firstly + as there is stack frame switching below, calling native in them + may cause register spilling work inproperly */ + if (!jit_emit_callnative(cc, os_time_get_boot_microsecond, time_started, + NULL, 0)) { + return NULL; + } +#endif +#endif + + /* top = exec_env->wasm_stack.s.top */ + GEN_INSN(LDPTR, top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top))); + /* top_boundary = exec_env->wasm_stack.s.top_boundary */ + GEN_INSN(LDPTR, top_boundary, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top_boundary))); + /* frame_boundary = top + frame_size + outs_size */ + GEN_INSN(ADD, frame_boundary, top, NEW_CONST(PTR, frame_size + outs_size)); + /* if frame_boundary > top_boundary, throw stack overflow exception */ + GEN_INSN(CMP, cc->cmp_reg, frame_boundary, top_boundary); + if (!jit_emit_exception(cc, EXCE_OPERAND_STACK_OVERFLOW, JIT_OP_BGTU, + cc->cmp_reg, NULL)) { + return NULL; + } + + /* Add first and then sub to reduce one used register */ + /* new_top = frame_boundary - outs_size = top + frame_size */ + GEN_INSN(SUB, new_top, frame_boundary, NEW_CONST(PTR, outs_size)); + /* exec_env->wasm_stack.s.top = new_top */ + GEN_INSN(STPTR, new_top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, wasm_stack.s.top))); + /* frame_sp = frame->lp + local_size */ + GEN_INSN(ADD, frame_sp, top, + NEW_CONST(PTR, offsetof(WASMInterpFrame, lp) + local_size)); + /* frame->sp = frame_sp */ + GEN_INSN(STPTR, frame_sp, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, sp))); + /* frame->prev_frame = fp_reg */ + GEN_INSN(STPTR, cc->fp_reg, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, prev_frame))); +#if WASM_ENABLE_DUMP_CALL_STACK != 0 || WASM_ENABLE_PERF_PROFILING != 0 + /* module_inst = exec_env->module_inst */ + GEN_INSN(LDPTR, module_inst, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, module_inst))); + func_insts_offset = + jit_frontend_get_module_inst_extra_offset(cur_wasm_module) + + (uint32)offsetof(WASMModuleInstanceExtra, functions); + /* func_inst = module_inst->e->functions */ + GEN_INSN(LDPTR, func_inst, module_inst, NEW_CONST(I32, func_insts_offset)); + /* func_inst = func_inst + cur_wasm_func_idx */ + GEN_INSN(ADD, func_inst, func_inst, + NEW_CONST(PTR, (uint32)sizeof(WASMFunctionInstance) + * cur_wasm_func_idx)); + /* frame->function = func_inst */ + GEN_INSN(STPTR, func_inst, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, function))); +#if WASM_ENABLE_PERF_PROFILING != 0 + /* frame->time_started = time_started */ + GEN_INSN(STI64, time_started, top, + NEW_CONST(I32, offsetof(WASMInterpFrame, time_started))); +#endif +#endif + /* exec_env->cur_frame = top */ + GEN_INSN(STPTR, top, cc->exec_env_reg, + NEW_CONST(I32, offsetof(WASMExecEnv, cur_frame))); + /* fp_reg = top */ + GEN_INSN(MOV, cc->fp_reg, top); + + /* Initialize local variables, set them to 0 */ + local_off = (uint32)offsetof(WASMInterpFrame, lp) + + cur_wasm_func->param_cell_num * 4; + for (i = 0; i < cur_wasm_func->local_cell_num / 2; i++, local_off += 8) { + GEN_INSN(STI64, NEW_CONST(I64, 0), cc->fp_reg, + NEW_CONST(I32, local_off)); + } + if (cur_wasm_func->local_cell_num & 1) { + GEN_INSN(STI32, NEW_CONST(I32, 0), cc->fp_reg, + NEW_CONST(I32, local_off)); + } + + return jit_frame; +} + +static void +free_block_memory(JitBlock *block) +{ + if (block->param_types) + jit_free(block->param_types); + if (block->result_types) + jit_free(block->result_types); + jit_free(block); +} + +static JitBasicBlock * +create_func_block(JitCompContext *cc) +{ + JitBlock *jit_block; + WASMFunction *cur_func = cc->cur_wasm_func; + WASMType *func_type = cur_func->func_type; + uint32 param_count = func_type->param_count; + uint32 result_count = func_type->result_count; + + if (!(jit_block = jit_calloc(sizeof(JitBlock)))) { + return NULL; + } + + if (param_count && !(jit_block->param_types = jit_calloc(param_count))) { + goto fail; + } + if (result_count && !(jit_block->result_types = jit_calloc(result_count))) { + goto fail; + } + + /* Set block data */ + jit_block->label_type = LABEL_TYPE_FUNCTION; + jit_block->param_count = param_count; + if (param_count) { + bh_memcpy_s(jit_block->param_types, param_count, func_type->types, + param_count); + } + jit_block->result_count = result_count; + if (result_count) { + bh_memcpy_s(jit_block->result_types, result_count, + func_type->types + param_count, result_count); + } + jit_block->wasm_code_end = cur_func->code + cur_func->code_size; + jit_block->frame_sp_begin = cc->jit_frame->sp; + + /* Add function entry block */ + if (!(jit_block->basic_block_entry = jit_cc_new_basic_block(cc, 0))) { + goto fail; + } + *(jit_annl_begin_bcip( + cc, jit_basic_block_label(jit_block->basic_block_entry))) = + cur_func->code; + jit_block_stack_push(&cc->block_stack, jit_block); + cc->cur_basic_block = jit_block->basic_block_entry; + + return jit_block->basic_block_entry; + +fail: + free_block_memory(jit_block); + return NULL; +} + +#define CHECK_BUF(buf, buf_end, length) \ + do { \ + if (buf + length > buf_end) { \ + jit_set_last_error(cc, "read leb failed: unexpected end."); \ + return false; \ + } \ + } while (0) + +static bool +read_leb(JitCompContext *cc, const uint8 *buf, const uint8 *buf_end, + uint32 *p_offset, uint32 maxbits, bool sign, uint64 *p_result) +{ + uint64 result = 0; + uint32 shift = 0; + uint32 bcnt = 0; + uint64 byte; + + while (true) { + CHECK_BUF(buf, buf_end, 1); + byte = buf[*p_offset]; + *p_offset += 1; + result |= ((byte & 0x7f) << shift); + shift += 7; + if ((byte & 0x80) == 0) { + break; + } + bcnt += 1; + } + if (bcnt > (maxbits + 6) / 7) { + jit_set_last_error(cc, "read leb failed: " + "integer representation too long"); + return false; + } + if (sign && (shift < maxbits) && (byte & 0x40)) { + /* Sign extend */ + result |= (~((uint64)0)) << shift; + } + *p_result = result; + return true; +} + +#define read_leb_uint32(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(cc, p, p_end, &off, 32, false, &res64)) \ + return false; \ + p += off; \ + res = (uint32)res64; \ + } while (0) + +#define read_leb_int32(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(cc, p, p_end, &off, 32, true, &res64)) \ + return false; \ + p += off; \ + res = (int32)res64; \ + } while (0) + +#define read_leb_int64(p, p_end, res) \ + do { \ + uint32 off = 0; \ + uint64 res64; \ + if (!read_leb(cc, p, p_end, &off, 64, true, &res64)) \ + return false; \ + p += off; \ + res = (int64)res64; \ + } while (0) + +#if WASM_ENABLE_SHARED_MEMORY != 0 +#define COMPILE_ATOMIC_RMW(OP, NAME) \ + case WASM_OP_ATOMIC_RMW_I32_##NAME: \ + bytes = 4; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME: \ + bytes = 8; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I32_##NAME##8_U: \ + bytes = 1; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I32_##NAME##16_U: \ + bytes = 2; \ + op_type = VALUE_TYPE_I32; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##8_U: \ + bytes = 1; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##16_U: \ + bytes = 2; \ + op_type = VALUE_TYPE_I64; \ + goto OP_ATOMIC_##OP; \ + case WASM_OP_ATOMIC_RMW_I64_##NAME##32_U: \ + bytes = 4; \ + op_type = VALUE_TYPE_I64; \ + OP_ATOMIC_##OP : bin_op = AtomicRMWBinOp##OP; \ + goto build_atomic_rmw; +#endif + +static bool +jit_compile_func(JitCompContext *cc) +{ + WASMFunction *cur_func = cc->cur_wasm_func; + WASMType *func_type = NULL; + uint8 *frame_ip = cur_func->code, opcode, *p_f32, *p_f64; + uint8 *frame_ip_end = frame_ip + cur_func->code_size; + uint8 *param_types = NULL, *result_types = NULL, value_type; + uint16 param_count, result_count; + uint32 br_depth, *br_depths, br_count; + uint32 func_idx, type_idx, mem_idx, local_idx, global_idx, i; + uint32 bytes = 4, align, offset; + bool merge_cmp_and_if = false, merge_cmp_and_br_if = false; + bool sign = true; + int32 i32_const; + int64 i64_const; + float32 f32_const; + float64 f64_const; + + while (frame_ip < frame_ip_end) { + cc->jit_frame->ip = frame_ip; + opcode = *frame_ip++; + +#if 0 /* TODO */ +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (cc->enable_thread_mgr) { + if (!check_suspend_flags(cc, func_ctx)) + return false; + } +#endif +#endif + + switch (opcode) { + case WASM_OP_UNREACHABLE: + if (!jit_compile_op_unreachable(cc, &frame_ip)) + return false; + break; + + case WASM_OP_NOP: + break; + + case WASM_OP_BLOCK: + case WASM_OP_LOOP: + case WASM_OP_IF: + { + value_type = *frame_ip++; + if (value_type == VALUE_TYPE_I32 || value_type == VALUE_TYPE_I64 + || value_type == VALUE_TYPE_F32 + || value_type == VALUE_TYPE_F64 + || value_type == VALUE_TYPE_V128 + || value_type == VALUE_TYPE_VOID + || value_type == VALUE_TYPE_FUNCREF + || value_type == VALUE_TYPE_EXTERNREF) { + param_count = 0; + param_types = NULL; + if (value_type == VALUE_TYPE_VOID) { + result_count = 0; + result_types = NULL; + } + else { + result_count = 1; + result_types = &value_type; + } + } + else { + jit_set_last_error(cc, "unsupported value type"); + return false; + } + if (!jit_compile_op_block( + cc, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - WASM_OP_BLOCK), + param_count, param_types, result_count, result_types, + merge_cmp_and_if)) + return false; + /* Clear flag */ + merge_cmp_and_if = false; + break; + } + case EXT_OP_BLOCK: + case EXT_OP_LOOP: + case EXT_OP_IF: + { + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + func_type = cc->cur_wasm_module->types[type_idx]; + param_count = func_type->param_count; + param_types = func_type->types; + result_count = func_type->result_count; + result_types = func_type->types + param_count; + if (!jit_compile_op_block( + cc, &frame_ip, frame_ip_end, + (uint32)(LABEL_TYPE_BLOCK + opcode - EXT_OP_BLOCK), + param_count, param_types, result_count, result_types, + merge_cmp_and_if)) + return false; + /* Clear flag */ + merge_cmp_and_if = false; + break; + } + + case WASM_OP_ELSE: + if (!jit_compile_op_else(cc, &frame_ip)) + return false; + break; + + case WASM_OP_END: + if (!jit_compile_op_end(cc, &frame_ip)) + return false; + break; + + case WASM_OP_BR: + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!jit_compile_op_br(cc, br_depth, &frame_ip)) + return false; + break; + + case WASM_OP_BR_IF: + read_leb_uint32(frame_ip, frame_ip_end, br_depth); + if (!jit_compile_op_br_if(cc, br_depth, merge_cmp_and_br_if, + &frame_ip)) + return false; + /* Clear flag */ + merge_cmp_and_br_if = false; + break; + + case WASM_OP_BR_TABLE: + read_leb_uint32(frame_ip, frame_ip_end, br_count); + if (!(br_depths = jit_calloc((uint32)sizeof(uint32) + * (br_count + 1)))) { + jit_set_last_error(cc, "allocate memory failed."); + goto fail; + } +#if WASM_ENABLE_FAST_INTERP != 0 + for (i = 0; i <= br_count; i++) + read_leb_uint32(frame_ip, frame_ip_end, br_depths[i]); +#else + for (i = 0; i <= br_count; i++) + br_depths[i] = *frame_ip++; +#endif + + if (!jit_compile_op_br_table(cc, br_depths, br_count, + &frame_ip)) { + jit_free(br_depths); + return false; + } + + jit_free(br_depths); + break; + +#if WASM_ENABLE_FAST_INTERP == 0 + case EXT_OP_BR_TABLE_CACHE: + { + BrTableCache *node = bh_list_first_elem( + cc->cur_wasm_module->br_table_cache_list); + BrTableCache *node_next; + uint8 *p_opcode = frame_ip - 1; + + read_leb_uint32(frame_ip, frame_ip_end, br_count); + + while (node) { + node_next = bh_list_elem_next(node); + if (node->br_table_op_addr == p_opcode) { + br_depths = node->br_depths; + if (!jit_compile_op_br_table(cc, br_depths, br_count, + &frame_ip)) { + return false; + } + break; + } + node = node_next; + } + bh_assert(node); + + break; + } +#endif + + case WASM_OP_RETURN: + if (!jit_compile_op_return(cc, &frame_ip)) + return false; + break; + + case WASM_OP_CALL: + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!jit_compile_op_call(cc, func_idx, false)) + return false; + break; + + case WASM_OP_CALL_INDIRECT: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); + +#if WASM_ENABLE_REF_TYPES != 0 + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); +#else + frame_ip++; + tbl_idx = 0; +#endif + + if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx)) + return false; + break; + } + +#if WASM_ENABLE_TAIL_CALL != 0 + case WASM_OP_RETURN_CALL: + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + + if (!jit_compile_op_call(cc, func_idx, true)) + return false; + if (!jit_compile_op_return(cc, &frame_ip)) + return false; + break; + + case WASM_OP_RETURN_CALL_INDIRECT: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, type_idx); +#if WASM_ENABLE_REF_TYPES != 0 + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); +#else + frame_ip++; + tbl_idx = 0; +#endif + + if (!jit_compile_op_call_indirect(cc, type_idx, tbl_idx)) + return false; + if (!jit_compile_op_return(cc, &frame_ip)) + return false; + break; + } +#endif /* end of WASM_ENABLE_TAIL_CALL */ + + case WASM_OP_DROP: + if (!jit_compile_op_drop(cc, true)) + return false; + break; + + case WASM_OP_DROP_64: + if (!jit_compile_op_drop(cc, false)) + return false; + break; + + case WASM_OP_SELECT: + if (!jit_compile_op_select(cc, true)) + return false; + break; + + case WASM_OP_SELECT_64: + if (!jit_compile_op_select(cc, false)) + return false; + break; + +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_SELECT_T: + { + uint32 vec_len; + + read_leb_uint32(frame_ip, frame_ip_end, vec_len); + bh_assert(vec_len == 1); + (void)vec_len; + + type_idx = *frame_ip++; + if (!jit_compile_op_select(cc, + (type_idx != VALUE_TYPE_I64) + && (type_idx != VALUE_TYPE_F64))) + return false; + break; + } + case WASM_OP_TABLE_GET: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_get(cc, tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_SET: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_set(cc, tbl_idx)) + return false; + break; + } + case WASM_OP_REF_NULL: + { + uint32 ref_type; + read_leb_uint32(frame_ip, frame_ip_end, ref_type); + if (!jit_compile_op_ref_null(cc, ref_type)) + return false; + break; + } + case WASM_OP_REF_IS_NULL: + { + if (!jit_compile_op_ref_is_null(cc)) + return false; + break; + } + case WASM_OP_REF_FUNC: + { + read_leb_uint32(frame_ip, frame_ip_end, func_idx); + if (!jit_compile_op_ref_func(cc, func_idx)) + return false; + break; + } +#endif + + case WASM_OP_GET_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!jit_compile_op_get_local(cc, local_idx)) + return false; + break; + + case WASM_OP_SET_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!jit_compile_op_set_local(cc, local_idx)) + return false; + break; + + case WASM_OP_TEE_LOCAL: + read_leb_uint32(frame_ip, frame_ip_end, local_idx); + if (!jit_compile_op_tee_local(cc, local_idx)) + return false; + break; + + case WASM_OP_GET_GLOBAL: + case WASM_OP_GET_GLOBAL_64: + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + if (!jit_compile_op_get_global(cc, global_idx)) + return false; + break; + + case WASM_OP_SET_GLOBAL: + case WASM_OP_SET_GLOBAL_64: + case WASM_OP_SET_GLOBAL_AUX_STACK: + read_leb_uint32(frame_ip, frame_ip_end, global_idx); + if (!jit_compile_op_set_global( + cc, global_idx, + opcode == WASM_OP_SET_GLOBAL_AUX_STACK ? true : false)) + return false; + break; + + case WASM_OP_I32_LOAD: + bytes = 4; + sign = true; + goto op_i32_load; + case WASM_OP_I32_LOAD8_S: + case WASM_OP_I32_LOAD8_U: + bytes = 1; + sign = (opcode == WASM_OP_I32_LOAD8_S) ? true : false; + goto op_i32_load; + case WASM_OP_I32_LOAD16_S: + case WASM_OP_I32_LOAD16_U: + bytes = 2; + sign = (opcode == WASM_OP_I32_LOAD16_S) ? true : false; + op_i32_load: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i32_load(cc, align, offset, bytes, sign, + false)) + return false; + break; + + case WASM_OP_I64_LOAD: + bytes = 8; + sign = true; + goto op_i64_load; + case WASM_OP_I64_LOAD8_S: + case WASM_OP_I64_LOAD8_U: + bytes = 1; + sign = (opcode == WASM_OP_I64_LOAD8_S) ? true : false; + goto op_i64_load; + case WASM_OP_I64_LOAD16_S: + case WASM_OP_I64_LOAD16_U: + bytes = 2; + sign = (opcode == WASM_OP_I64_LOAD16_S) ? true : false; + goto op_i64_load; + case WASM_OP_I64_LOAD32_S: + case WASM_OP_I64_LOAD32_U: + bytes = 4; + sign = (opcode == WASM_OP_I64_LOAD32_S) ? true : false; + op_i64_load: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i64_load(cc, align, offset, bytes, sign, + false)) + return false; + break; + + case WASM_OP_F32_LOAD: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f32_load(cc, align, offset)) + return false; + break; + + case WASM_OP_F64_LOAD: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f64_load(cc, align, offset)) + return false; + break; + + case WASM_OP_I32_STORE: + bytes = 4; + goto op_i32_store; + case WASM_OP_I32_STORE8: + bytes = 1; + goto op_i32_store; + case WASM_OP_I32_STORE16: + bytes = 2; + op_i32_store: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i32_store(cc, align, offset, bytes, false)) + return false; + break; + + case WASM_OP_I64_STORE: + bytes = 8; + goto op_i64_store; + case WASM_OP_I64_STORE8: + bytes = 1; + goto op_i64_store; + case WASM_OP_I64_STORE16: + bytes = 2; + goto op_i64_store; + case WASM_OP_I64_STORE32: + bytes = 4; + op_i64_store: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_i64_store(cc, align, offset, bytes, false)) + return false; + break; + + case WASM_OP_F32_STORE: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f32_store(cc, align, offset)) + return false; + break; + + case WASM_OP_F64_STORE: + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + if (!jit_compile_op_f64_store(cc, align, offset)) + return false; + break; + + case WASM_OP_MEMORY_SIZE: + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_size(cc, mem_idx)) + return false; + break; + + case WASM_OP_MEMORY_GROW: + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_grow(cc, mem_idx)) + return false; + break; + + case WASM_OP_I32_CONST: + read_leb_int32(frame_ip, frame_ip_end, i32_const); + if (!jit_compile_op_i32_const(cc, i32_const)) + return false; + break; + + case WASM_OP_I64_CONST: + read_leb_int64(frame_ip, frame_ip_end, i64_const); + if (!jit_compile_op_i64_const(cc, i64_const)) + return false; + break; + + case WASM_OP_F32_CONST: + p_f32 = (uint8 *)&f32_const; + for (i = 0; i < sizeof(float32); i++) + *p_f32++ = *frame_ip++; + if (!jit_compile_op_f32_const(cc, f32_const)) + return false; + break; + + case WASM_OP_F64_CONST: + p_f64 = (uint8 *)&f64_const; + for (i = 0; i < sizeof(float64); i++) + *p_f64++ = *frame_ip++; + if (!jit_compile_op_f64_const(cc, f64_const)) + return false; + break; + + case WASM_OP_I32_EQZ: + case WASM_OP_I32_EQ: + case WASM_OP_I32_NE: + case WASM_OP_I32_LT_S: + case WASM_OP_I32_LT_U: + case WASM_OP_I32_GT_S: + case WASM_OP_I32_GT_U: + case WASM_OP_I32_LE_S: + case WASM_OP_I32_LE_U: + case WASM_OP_I32_GE_S: + case WASM_OP_I32_GE_U: + if (!jit_compile_op_i32_compare(cc, INT_EQZ + opcode + - WASM_OP_I32_EQZ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_I64_EQZ: + case WASM_OP_I64_EQ: + case WASM_OP_I64_NE: + case WASM_OP_I64_LT_S: + case WASM_OP_I64_LT_U: + case WASM_OP_I64_GT_S: + case WASM_OP_I64_GT_U: + case WASM_OP_I64_LE_S: + case WASM_OP_I64_LE_U: + case WASM_OP_I64_GE_S: + case WASM_OP_I64_GE_U: + if (!jit_compile_op_i64_compare(cc, INT_EQZ + opcode + - WASM_OP_I64_EQZ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_F32_EQ: + case WASM_OP_F32_NE: + case WASM_OP_F32_LT: + case WASM_OP_F32_GT: + case WASM_OP_F32_LE: + case WASM_OP_F32_GE: + if (!jit_compile_op_f32_compare(cc, FLOAT_EQ + opcode + - WASM_OP_F32_EQ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_F64_EQ: + case WASM_OP_F64_NE: + case WASM_OP_F64_LT: + case WASM_OP_F64_GT: + case WASM_OP_F64_LE: + case WASM_OP_F64_GE: + if (!jit_compile_op_f64_compare(cc, FLOAT_EQ + opcode + - WASM_OP_F64_EQ)) + return false; + if (frame_ip < frame_ip_end) { + /* Merge `CMP, SELECTcc, CMP, BNE` insns into `CMP, Bcc` */ + if (*frame_ip == WASM_OP_IF || *frame_ip == EXT_OP_IF) + merge_cmp_and_if = true; + if (*frame_ip == WASM_OP_BR_IF) + merge_cmp_and_br_if = true; + } + break; + + case WASM_OP_I32_CLZ: + if (!jit_compile_op_i32_clz(cc)) + return false; + break; + + case WASM_OP_I32_CTZ: + if (!jit_compile_op_i32_ctz(cc)) + return false; + break; + + case WASM_OP_I32_POPCNT: + if (!jit_compile_op_i32_popcnt(cc)) + return false; + break; + + case WASM_OP_I32_ADD: + case WASM_OP_I32_SUB: + case WASM_OP_I32_MUL: + case WASM_OP_I32_DIV_S: + case WASM_OP_I32_DIV_U: + case WASM_OP_I32_REM_S: + case WASM_OP_I32_REM_U: + if (!jit_compile_op_i32_arithmetic( + cc, INT_ADD + opcode - WASM_OP_I32_ADD, &frame_ip)) + return false; + break; + + case WASM_OP_I32_AND: + case WASM_OP_I32_OR: + case WASM_OP_I32_XOR: + if (!jit_compile_op_i32_bitwise(cc, INT_SHL + opcode + - WASM_OP_I32_AND)) + return false; + break; + + case WASM_OP_I32_SHL: + case WASM_OP_I32_SHR_S: + case WASM_OP_I32_SHR_U: + case WASM_OP_I32_ROTL: + case WASM_OP_I32_ROTR: + if (!jit_compile_op_i32_shift(cc, INT_SHL + opcode + - WASM_OP_I32_SHL)) + return false; + break; + + case WASM_OP_I64_CLZ: + if (!jit_compile_op_i64_clz(cc)) + return false; + break; + + case WASM_OP_I64_CTZ: + if (!jit_compile_op_i64_ctz(cc)) + return false; + break; + + case WASM_OP_I64_POPCNT: + if (!jit_compile_op_i64_popcnt(cc)) + return false; + break; + + case WASM_OP_I64_ADD: + case WASM_OP_I64_SUB: + case WASM_OP_I64_MUL: + case WASM_OP_I64_DIV_S: + case WASM_OP_I64_DIV_U: + case WASM_OP_I64_REM_S: + case WASM_OP_I64_REM_U: + if (!jit_compile_op_i64_arithmetic( + cc, INT_ADD + opcode - WASM_OP_I64_ADD, &frame_ip)) + return false; + break; + + case WASM_OP_I64_AND: + case WASM_OP_I64_OR: + case WASM_OP_I64_XOR: + if (!jit_compile_op_i64_bitwise(cc, INT_SHL + opcode + - WASM_OP_I64_AND)) + return false; + break; + + case WASM_OP_I64_SHL: + case WASM_OP_I64_SHR_S: + case WASM_OP_I64_SHR_U: + case WASM_OP_I64_ROTL: + case WASM_OP_I64_ROTR: + if (!jit_compile_op_i64_shift(cc, INT_SHL + opcode + - WASM_OP_I64_SHL)) + return false; + break; + + case WASM_OP_F32_ABS: + case WASM_OP_F32_NEG: + case WASM_OP_F32_CEIL: + case WASM_OP_F32_FLOOR: + case WASM_OP_F32_TRUNC: + case WASM_OP_F32_NEAREST: + case WASM_OP_F32_SQRT: + if (!jit_compile_op_f32_math(cc, FLOAT_ABS + opcode + - WASM_OP_F32_ABS)) + return false; + break; + + case WASM_OP_F32_ADD: + case WASM_OP_F32_SUB: + case WASM_OP_F32_MUL: + case WASM_OP_F32_DIV: + case WASM_OP_F32_MIN: + case WASM_OP_F32_MAX: + if (!jit_compile_op_f32_arithmetic(cc, FLOAT_ADD + opcode + - WASM_OP_F32_ADD)) + return false; + break; + + case WASM_OP_F32_COPYSIGN: + if (!jit_compile_op_f32_copysign(cc)) + return false; + break; + + case WASM_OP_F64_ABS: + case WASM_OP_F64_NEG: + case WASM_OP_F64_CEIL: + case WASM_OP_F64_FLOOR: + case WASM_OP_F64_TRUNC: + case WASM_OP_F64_NEAREST: + case WASM_OP_F64_SQRT: + if (!jit_compile_op_f64_math(cc, FLOAT_ABS + opcode + - WASM_OP_F64_ABS)) + return false; + break; + + case WASM_OP_F64_ADD: + case WASM_OP_F64_SUB: + case WASM_OP_F64_MUL: + case WASM_OP_F64_DIV: + case WASM_OP_F64_MIN: + case WASM_OP_F64_MAX: + if (!jit_compile_op_f64_arithmetic(cc, FLOAT_ADD + opcode + - WASM_OP_F64_ADD)) + return false; + break; + + case WASM_OP_F64_COPYSIGN: + if (!jit_compile_op_f64_copysign(cc)) + return false; + break; + + case WASM_OP_I32_WRAP_I64: + if (!jit_compile_op_i32_wrap_i64(cc)) + return false; + break; + + case WASM_OP_I32_TRUNC_S_F32: + case WASM_OP_I32_TRUNC_U_F32: + sign = (opcode == WASM_OP_I32_TRUNC_S_F32) ? true : false; + if (!jit_compile_op_i32_trunc_f32(cc, sign, false)) + return false; + break; + + case WASM_OP_I32_TRUNC_S_F64: + case WASM_OP_I32_TRUNC_U_F64: + sign = (opcode == WASM_OP_I32_TRUNC_S_F64) ? true : false; + if (!jit_compile_op_i32_trunc_f64(cc, sign, false)) + return false; + break; + + case WASM_OP_I64_EXTEND_S_I32: + case WASM_OP_I64_EXTEND_U_I32: + sign = (opcode == WASM_OP_I64_EXTEND_S_I32) ? true : false; + if (!jit_compile_op_i64_extend_i32(cc, sign)) + return false; + break; + + case WASM_OP_I64_TRUNC_S_F32: + case WASM_OP_I64_TRUNC_U_F32: + sign = (opcode == WASM_OP_I64_TRUNC_S_F32) ? true : false; + if (!jit_compile_op_i64_trunc_f32(cc, sign, false)) + return false; + break; + + case WASM_OP_I64_TRUNC_S_F64: + case WASM_OP_I64_TRUNC_U_F64: + sign = (opcode == WASM_OP_I64_TRUNC_S_F64) ? true : false; + if (!jit_compile_op_i64_trunc_f64(cc, sign, false)) + return false; + break; + + case WASM_OP_F32_CONVERT_S_I32: + case WASM_OP_F32_CONVERT_U_I32: + sign = (opcode == WASM_OP_F32_CONVERT_S_I32) ? true : false; + if (!jit_compile_op_f32_convert_i32(cc, sign)) + return false; + break; + + case WASM_OP_F32_CONVERT_S_I64: + case WASM_OP_F32_CONVERT_U_I64: + sign = (opcode == WASM_OP_F32_CONVERT_S_I64) ? true : false; + if (!jit_compile_op_f32_convert_i64(cc, sign)) + return false; + break; + + case WASM_OP_F32_DEMOTE_F64: + if (!jit_compile_op_f32_demote_f64(cc)) + return false; + break; + + case WASM_OP_F64_CONVERT_S_I32: + case WASM_OP_F64_CONVERT_U_I32: + sign = (opcode == WASM_OP_F64_CONVERT_S_I32) ? true : false; + if (!jit_compile_op_f64_convert_i32(cc, sign)) + return false; + break; + + case WASM_OP_F64_CONVERT_S_I64: + case WASM_OP_F64_CONVERT_U_I64: + sign = (opcode == WASM_OP_F64_CONVERT_S_I64) ? true : false; + if (!jit_compile_op_f64_convert_i64(cc, sign)) + return false; + break; + + case WASM_OP_F64_PROMOTE_F32: + if (!jit_compile_op_f64_promote_f32(cc)) + return false; + break; + + case WASM_OP_I32_REINTERPRET_F32: + if (!jit_compile_op_i32_reinterpret_f32(cc)) + return false; + break; + + case WASM_OP_I64_REINTERPRET_F64: + if (!jit_compile_op_i64_reinterpret_f64(cc)) + return false; + break; + + case WASM_OP_F32_REINTERPRET_I32: + if (!jit_compile_op_f32_reinterpret_i32(cc)) + return false; + break; + + case WASM_OP_F64_REINTERPRET_I64: + if (!jit_compile_op_f64_reinterpret_i64(cc)) + return false; + break; + + case WASM_OP_I32_EXTEND8_S: + if (!jit_compile_op_i32_extend_i32(cc, 8)) + return false; + break; + + case WASM_OP_I32_EXTEND16_S: + if (!jit_compile_op_i32_extend_i32(cc, 16)) + return false; + break; + + case WASM_OP_I64_EXTEND8_S: + if (!jit_compile_op_i64_extend_i64(cc, 8)) + return false; + break; + + case WASM_OP_I64_EXTEND16_S: + if (!jit_compile_op_i64_extend_i64(cc, 16)) + return false; + break; + + case WASM_OP_I64_EXTEND32_S: + if (!jit_compile_op_i64_extend_i64(cc, 32)) + return false; + break; + + case WASM_OP_MISC_PREFIX: + { + uint32 opcode1; + + read_leb_uint32(frame_ip, frame_ip_end, opcode1); + opcode = (uint32)opcode1; + + switch (opcode) { + case WASM_OP_I32_TRUNC_SAT_S_F32: + case WASM_OP_I32_TRUNC_SAT_U_F32: + sign = (opcode == WASM_OP_I32_TRUNC_SAT_S_F32) ? true + : false; + if (!jit_compile_op_i32_trunc_f32(cc, sign, true)) + return false; + break; + case WASM_OP_I32_TRUNC_SAT_S_F64: + case WASM_OP_I32_TRUNC_SAT_U_F64: + sign = (opcode == WASM_OP_I32_TRUNC_SAT_S_F64) ? true + : false; + if (!jit_compile_op_i32_trunc_f64(cc, sign, true)) + return false; + break; + case WASM_OP_I64_TRUNC_SAT_S_F32: + case WASM_OP_I64_TRUNC_SAT_U_F32: + sign = (opcode == WASM_OP_I64_TRUNC_SAT_S_F32) ? true + : false; + if (!jit_compile_op_i64_trunc_f32(cc, sign, true)) + return false; + break; + case WASM_OP_I64_TRUNC_SAT_S_F64: + case WASM_OP_I64_TRUNC_SAT_U_F64: + sign = (opcode == WASM_OP_I64_TRUNC_SAT_S_F64) ? true + : false; + if (!jit_compile_op_i64_trunc_f64(cc, sign, true)) + return false; + break; +#if WASM_ENABLE_BULK_MEMORY != 0 + case WASM_OP_MEMORY_INIT: + { + uint32 seg_idx = 0; + read_leb_uint32(frame_ip, frame_ip_end, seg_idx); + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_init(cc, mem_idx, seg_idx)) + return false; + break; + } + case WASM_OP_DATA_DROP: + { + uint32 seg_idx; + read_leb_uint32(frame_ip, frame_ip_end, seg_idx); + if (!jit_compile_op_data_drop(cc, seg_idx)) + return false; + break; + } + case WASM_OP_MEMORY_COPY: + { + uint32 src_mem_idx, dst_mem_idx; + read_leb_uint32(frame_ip, frame_ip_end, src_mem_idx); + read_leb_uint32(frame_ip, frame_ip_end, dst_mem_idx); + if (!jit_compile_op_memory_copy(cc, src_mem_idx, + dst_mem_idx)) + return false; + break; + } + case WASM_OP_MEMORY_FILL: + { + read_leb_uint32(frame_ip, frame_ip_end, mem_idx); + if (!jit_compile_op_memory_fill(cc, mem_idx)) + return false; + break; + } +#endif /* WASM_ENABLE_BULK_MEMORY */ +#if WASM_ENABLE_REF_TYPES != 0 + case WASM_OP_TABLE_INIT: + { + uint32 tbl_idx, tbl_seg_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_seg_idx); + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_init(cc, tbl_idx, + tbl_seg_idx)) + return false; + break; + } + case WASM_OP_ELEM_DROP: + { + uint32 tbl_seg_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_seg_idx); + if (!jit_compile_op_elem_drop(cc, tbl_seg_idx)) + return false; + break; + } + case WASM_OP_TABLE_COPY: + { + uint32 src_tbl_idx, dst_tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, dst_tbl_idx); + read_leb_uint32(frame_ip, frame_ip_end, src_tbl_idx); + if (!jit_compile_op_table_copy(cc, src_tbl_idx, + dst_tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_GROW: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_grow(cc, tbl_idx)) + return false; + break; + } + + case WASM_OP_TABLE_SIZE: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_size(cc, tbl_idx)) + return false; + break; + } + case WASM_OP_TABLE_FILL: + { + uint32 tbl_idx; + + read_leb_uint32(frame_ip, frame_ip_end, tbl_idx); + if (!jit_compile_op_table_fill(cc, tbl_idx)) + return false; + break; + } +#endif /* WASM_ENABLE_REF_TYPES */ + default: + jit_set_last_error(cc, "unsupported opcode"); + return false; + } + break; + } + +#if WASM_ENABLE_SHARED_MEMORY != 0 + case WASM_OP_ATOMIC_PREFIX: + { + uint8 bin_op, op_type; + + if (frame_ip < frame_ip_end) { + opcode = *frame_ip++; + } + if (opcode != WASM_OP_ATOMIC_FENCE) { + read_leb_uint32(frame_ip, frame_ip_end, align); + read_leb_uint32(frame_ip, frame_ip_end, offset); + } + switch (opcode) { + case WASM_OP_ATOMIC_WAIT32: + if (!jit_compile_op_atomic_wait(cc, VALUE_TYPE_I32, + align, offset, 4)) + return false; + break; + case WASM_OP_ATOMIC_WAIT64: + if (!jit_compile_op_atomic_wait(cc, VALUE_TYPE_I64, + align, offset, 8)) + return false; + break; + case WASM_OP_ATOMIC_NOTIFY: + if (!jit_compiler_op_atomic_notify(cc, align, offset, + bytes)) + return false; + break; + case WASM_OP_ATOMIC_FENCE: + /* Skip memory index */ + frame_ip++; + if (!jit_compiler_op_atomic_fence(cc)) + return false; + break; + case WASM_OP_ATOMIC_I32_LOAD: + bytes = 4; + goto op_atomic_i32_load; + case WASM_OP_ATOMIC_I32_LOAD8_U: + bytes = 1; + goto op_atomic_i32_load; + case WASM_OP_ATOMIC_I32_LOAD16_U: + bytes = 2; + op_atomic_i32_load: + if (!jit_compile_op_i32_load(cc, align, offset, bytes, + sign, true)) + return false; + break; + + case WASM_OP_ATOMIC_I64_LOAD: + bytes = 8; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD8_U: + bytes = 1; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD16_U: + bytes = 2; + goto op_atomic_i64_load; + case WASM_OP_ATOMIC_I64_LOAD32_U: + bytes = 4; + op_atomic_i64_load: + if (!jit_compile_op_i64_load(cc, align, offset, bytes, + sign, true)) + return false; + break; + + case WASM_OP_ATOMIC_I32_STORE: + bytes = 4; + goto op_atomic_i32_store; + case WASM_OP_ATOMIC_I32_STORE8: + bytes = 1; + goto op_atomic_i32_store; + case WASM_OP_ATOMIC_I32_STORE16: + bytes = 2; + op_atomic_i32_store: + if (!jit_compile_op_i32_store(cc, align, offset, bytes, + true)) + return false; + break; + + case WASM_OP_ATOMIC_I64_STORE: + bytes = 8; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE8: + bytes = 1; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE16: + bytes = 2; + goto op_atomic_i64_store; + case WASM_OP_ATOMIC_I64_STORE32: + bytes = 4; + op_atomic_i64_store: + if (!jit_compile_op_i64_store(cc, align, offset, bytes, + true)) + return false; + break; + + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG: + bytes = 4; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG: + bytes = 8; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG8_U: + bytes = 1; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I32_CMPXCHG16_U: + bytes = 2; + op_type = VALUE_TYPE_I32; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG8_U: + bytes = 1; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG16_U: + bytes = 2; + op_type = VALUE_TYPE_I64; + goto op_atomic_cmpxchg; + case WASM_OP_ATOMIC_RMW_I64_CMPXCHG32_U: + bytes = 4; + op_type = VALUE_TYPE_I64; + op_atomic_cmpxchg: + if (!jit_compile_op_atomic_cmpxchg(cc, op_type, align, + offset, bytes)) + return false; + break; + + COMPILE_ATOMIC_RMW(Add, ADD); + COMPILE_ATOMIC_RMW(Sub, SUB); + COMPILE_ATOMIC_RMW(And, AND); + COMPILE_ATOMIC_RMW(Or, OR); + COMPILE_ATOMIC_RMW(Xor, XOR); + COMPILE_ATOMIC_RMW(Xchg, XCHG); + + build_atomic_rmw: + if (!jit_compile_op_atomic_rmw(cc, bin_op, op_type, + align, offset, bytes)) + return false; + break; + + default: + jit_set_last_error(cc, "unsupported opcode"); + return false; + } + break; + } +#endif /* end of WASM_ENABLE_SHARED_MEMORY */ + + default: + jit_set_last_error(cc, "unsupported opcode"); + return false; + } + /* Error may occur when creating registers, basic blocks, insns, + consts and labels, in which the return value may be unchecked, + here we check again */ + if (jit_get_last_error(cc)) { + return false; + } + } + + (void)func_idx; + return true; +fail: + return false; +} + +JitBasicBlock * +jit_frontend_translate_func(JitCompContext *cc) +{ + JitFrame *jit_frame; + JitBasicBlock *basic_block_entry; + + if (!(jit_frame = init_func_translation(cc))) { + return NULL; + } + + if (!(basic_block_entry = create_func_block(cc))) { + return NULL; + } + + if (!jit_compile_func(cc)) { + return NULL; + } + + return basic_block_entry; +} + +uint32 +jit_frontend_get_jitted_return_addr_offset() +{ + return (uint32)offsetof(WASMInterpFrame, jitted_return_addr); +} + +#if 0 +#if WASM_ENABLE_THREAD_MGR != 0 +bool +check_suspend_flags(JitCompContext *cc, JITFuncContext *func_ctx) +{ + LLVMValueRef terminate_addr, terminate_flags, flag, offset, res; + JitBasicBlock *terminate_check_block, non_terminate_block; + JITFuncType *jit_func_type = func_ctx->jit_func->func_type; + JitBasicBlock *terminate_block; + + /* Offset of suspend_flags */ + offset = I32_FIVE; + + if (!(terminate_addr = LLVMBuildInBoundsGEP( + cc->builder, func_ctx->exec_env, &offset, 1, "terminate_addr"))) { + jit_set_last_error("llvm build in bounds gep failed"); + return false; + } + if (!(terminate_addr = + LLVMBuildBitCast(cc->builder, terminate_addr, INT32_PTR_TYPE, + "terminate_addr_ptr"))) { + jit_set_last_error("llvm build bit cast failed"); + return false; + } + + if (!(terminate_flags = + LLVMBuildLoad(cc->builder, terminate_addr, "terminate_flags"))) { + jit_set_last_error("llvm build bit cast failed"); + return false; + } + /* Set terminate_flags memory accecc to volatile, so that the value + will always be loaded from memory rather than register */ + LLVMSetVolatile(terminate_flags, true); + + CREATE_BASIC_BLOCK(terminate_check_block, "terminate_check"); + MOVE_BASIC_BLOCK_AFTER_CURR(terminate_check_block); + + CREATE_BASIC_BLOCK(non_terminate_block, "non_terminate"); + MOVE_BASIC_BLOCK_AFTER_CURR(non_terminate_block); + + BUILD_ICMP(LLVMIntSGT, terminate_flags, I32_ZERO, res, "need_terminate"); + BUILD_COND_BR(res, terminate_check_block, non_terminate_block); + + /* Move builder to terminate check block */ + SET_BUILDER_POS(terminate_check_block); + + CREATE_BASIC_BLOCK(terminate_block, "terminate"); + MOVE_BASIC_BLOCK_AFTER_CURR(terminate_block); + + if (!(flag = LLVMBuildAnd(cc->builder, terminate_flags, I32_ONE, + "termination_flag"))) { + jit_set_last_error("llvm build AND failed"); + return false; + } + + BUILD_ICMP(LLVMIntSGT, flag, I32_ZERO, res, "need_terminate"); + BUILD_COND_BR(res, terminate_block, non_terminate_block); + + /* Move builder to terminate block */ + SET_BUILDER_POS(terminate_block); + if (!jit_build_zero_function_ret(cc, func_ctx, jit_func_type)) { + goto fail; + } + + /* Move builder to terminate block */ + SET_BUILDER_POS(non_terminate_block); + return true; + +fail: + return false; +} +#endif /* End of WASM_ENABLE_THREAD_MGR */ +#endif -- cgit v1.2.3