From 788cbf2a1945cf04cc8139f7a363e268f18cbe34 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Tue, 5 Jan 2021 04:05:30 -0600 Subject: [PATCH] Refine aot call_indirect opcode translation (#492) Re-implement aot call_indirect opcode translation: when calling non-import function, translate it by LLVM call IR to call the function in AOTed code, so as to avoid calling runtime aot_call_indirect API which is much slower. For import function, keep calling aot_call_indirect API due to the possible pointer/string argument conversion. And add prompt info while app heap is corrupted, change emit_leb to emit_uint32 inter fast-interp to refine footprint. Signed-off-by: Wenyong Huang --- core/iwasm/aot/aot_runtime.c | 27 +- core/iwasm/aot/aot_runtime.h | 1 - core/iwasm/common/wasm_runtime_common.c | 7 +- core/iwasm/compilation/aot_emit_function.c | 466 ++++++++++++++++++--- core/iwasm/compilation/aot_llvm.c | 45 ++ core/iwasm/compilation/aot_llvm.h | 2 +- core/iwasm/interpreter/wasm.h | 13 + core/iwasm/interpreter/wasm_interp_fast.c | 39 +- core/iwasm/interpreter/wasm_loader.c | 38 +- core/iwasm/interpreter/wasm_mini_loader.c | 27 +- core/iwasm/interpreter/wasm_runtime.c | 12 +- core/shared/mem-alloc/ems/ems_gc.h | 10 +- core/shared/mem-alloc/ems/ems_kfc.c | 9 +- core/shared/mem-alloc/mem_alloc.c | 20 +- core/shared/mem-alloc/mem_alloc.h | 4 +- 15 files changed, 529 insertions(+), 191 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 7b5df64d..d26f2161 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1381,7 +1381,17 @@ aot_module_malloc(AOTModuleInstance *module_inst, uint32 size, } if (!addr) { - aot_set_exception(module_inst, "out of memory"); + if (memory_inst->heap_handle.ptr + && mem_allocator_is_heap_corrupted(memory_inst->heap_handle.ptr)) { + LOG_ERROR("Error: app heap is corrupted, if the wasm file " + "is compiled by wasi-sdk-12.0 or larger version, " + "please add -Wl,--export=malloc -Wl,--export=free " + " to export malloc and free functions."); + aot_set_exception(module_inst, "app heap corrupted"); + } + else { + aot_set_exception(module_inst, "out of memory"); + } return 0; } if (p_native_addr) @@ -1804,7 +1814,6 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, bool aot_call_indirect(WASMExecEnv *exec_env, - bool check_func_type, uint32 func_type_idx, uint32 table_elem_idx, uint32 argc, uint32 *argv) { @@ -1816,8 +1825,7 @@ aot_call_indirect(WASMExecEnv *exec_env, AOTFuncType *func_type; void **func_ptrs = (void**)module_inst->func_ptrs.ptr, *func_ptr; uint32 table_size = module_inst->table_size; - uint32 func_idx, func_type_idx1; - uint32 ext_ret_count; + uint32 func_type_idx, func_idx, ext_ret_count; AOTImportFunc *import_func; const char *signature = NULL; void *attachment = NULL; @@ -1844,15 +1852,8 @@ aot_call_indirect(WASMExecEnv *exec_env, return false; } - func_type_idx1 = func_type_indexes[func_idx]; - if (check_func_type - && !aot_is_wasm_type_equal(module_inst, func_type_idx, - func_type_idx1)) { - aot_set_exception_with_id(module_inst, - EXCE_INVALID_FUNCTION_TYPE_INDEX); - return false; - } - func_type = aot_module->func_types[func_type_idx1]; + func_type_idx = func_type_indexes[func_idx]; + func_type = aot_module->func_types[func_type_idx]; if (!(func_ptr = func_ptrs[func_idx])) { bh_assert(func_idx < aot_module->import_func_count); diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 87dc1166..4fab1903 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -521,7 +521,6 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, bool aot_call_indirect(WASMExecEnv *exec_env, - bool check_func_type, uint32 func_type_idx, uint32 table_elem_idx, uint32 argc, uint32 *argv); diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 45b81b9d..9c8b3a06 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -3327,14 +3327,11 @@ wasm_runtime_call_indirect(WASMExecEnv *exec_env, #if WASM_ENABLE_INTERP != 0 if (exec_env->module_inst->module_type == Wasm_Module_Bytecode) - return wasm_call_indirect(exec_env, - element_indices, - argc, argv); + return wasm_call_indirect(exec_env, element_indices, argc, argv); #endif #if WASM_ENABLE_AOT != 0 if (exec_env->module_inst->module_type == Wasm_Module_AoT) - return aot_call_indirect(exec_env, false, 0, - element_indices, argc, argv); + return aot_call_indirect(exec_env, element_indices, argc, argv); #endif return false; } diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index ade942d5..1aa9eb57 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -347,7 +347,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + ext_ret_count); if (total_size >= UINT32_MAX || !(param_values = wasm_runtime_malloc((uint32)total_size))) { - aot_set_last_error("Allocate memory failed."); + aot_set_last_error("allocate memory failed."); return false; } @@ -408,7 +408,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, total_size = sizeof(LLVMTypeRef) * (uint64)(param_count + 1); if (total_size >= UINT32_MAX || !(param_types = wasm_runtime_malloc((uint32)total_size))) { - aot_set_last_error("Allocate memory failed."); + aot_set_last_error("allocate memory failed."); goto fail; } @@ -510,22 +510,12 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, char buf[32], *func_name = "aot_call_indirect"; uint32 i, cell_num = 0, ret_cell_num, argv_cell_num; -#if WASM_ENABLE_THREAD_MGR != 0 - /* Insert suspend check point */ - if (comp_ctx->enable_thread_mgr) { - if (!check_suspend_flags(comp_ctx, func_ctx)) - return false; - } -#endif - /* prepare function type of aot_call_indirect */ func_param_types[0] = comp_ctx->exec_env_type; /* exec_env */ - func_param_types[1] = INT8_TYPE; /* check_func_type */ - func_param_types[2] = I32_TYPE; /* func_type_idx */ - func_param_types[3] = I32_TYPE; /* table_elem_idx */ - func_param_types[4] = I32_TYPE; /* argc */ - func_param_types[5] = INT32_PTR_TYPE; /* argv */ - if (!(func_type = LLVMFunctionType(INT8_TYPE, func_param_types, 6, false))) { + func_param_types[1] = I32_TYPE; /* table_elem_idx */ + func_param_types[2] = I32_TYPE; /* argc */ + func_param_types[3] = INT32_PTR_TYPE; /* argv */ + if (!(func_type = LLVMFunctionType(INT8_TYPE, func_param_types, 4, false))) { aot_set_last_error("llvm add function type failed."); return false; } @@ -588,20 +578,18 @@ call_aot_call_indirect_func(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } func_param_values[0] = func_ctx->exec_env; - func_param_values[1] = I8_CONST(true); - func_param_values[2] = func_type_idx; - func_param_values[3] = table_elem_idx; - func_param_values[4] = I32_CONST(param_cell_num); - func_param_values[5] = func_ctx->argv_buf; + func_param_values[1] = table_elem_idx; + func_param_values[2] = I32_CONST(param_cell_num); + func_param_values[3] = func_ctx->argv_buf; - if (!func_param_values[1] || !func_param_values[4]) { + if (!func_param_values[2]) { aot_set_last_error("llvm create const failed."); return false; } /* call aot_call_indirect() function */ if (!(res = LLVMBuildCall(comp_ctx->builder, func, - func_param_values, 6, "res"))) { + func_param_values, 4, "res"))) { aot_set_last_error("llvm build call failed."); return false; } @@ -642,89 +630,453 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 type_idx) { AOTFuncType *func_type; - LLVMValueRef elem_idx, ftype_idx; - LLVMValueRef *param_values = NULL, *value_rets = NULL, res = NULL; - LLVMTypeRef *param_types = NULL; - int32 i, param_count, result_count; - uint32 param_cell_num; + LLVMValueRef elem_idx, table_elem, func_idx; + LLVMValueRef ftype_idx_ptr, ftype_idx, ftype_idx_const; + LLVMValueRef cmp_elem_idx, cmp_func_idx, cmp_ftype_idx; + LLVMValueRef func, func_ptr, table_size_const; + LLVMValueRef ext_ret_offset, ext_ret_ptr, ext_ret, res; + LLVMValueRef *param_values = NULL, *value_rets = NULL; + LLVMValueRef *result_phis = NULL, value_ret, import_func_count; + LLVMTypeRef *param_types = NULL, ret_type; + LLVMTypeRef llvm_func_type, llvm_func_ptr_type; + LLVMTypeRef ext_ret_ptr_type; + LLVMBasicBlockRef check_elem_idx_succ, check_ftype_idx_succ; + LLVMBasicBlockRef check_func_idx_succ, block_return, block_curr; + LLVMBasicBlockRef block_call_import, block_call_non_import; + uint32 total_param_count, func_param_count, func_result_count; + uint32 table_init_size = 0, ext_cell_num, param_cell_num, i, j; + uint8 wasm_ret_type, *wasm_ret_types; uint64 total_size; - uint8 *wasm_ret_types = NULL; - bool ret = false; + char buf[32]; + bool ret; /* Check function type index */ if (type_idx >= comp_ctx->comp_data->func_type_count) { - aot_set_last_error("type index is overflow"); + aot_set_last_error("function type index out of range"); return false; } - ftype_idx = I32_CONST(type_idx); - CHECK_LLVM_CONST(ftype_idx); + /* Find the equivalent function type whose type index is the smallest: + the callee function's type index is also converted to the smallest + one in wasm loader, so we can just check whether the two type indexes + are equal (the type index of call_indirect opcode and callee func), + we don't need to check whether the whole function types are equal, + including param types and result types. */ + type_idx = wasm_get_smallest_type_idx(comp_ctx->comp_data->func_types, + comp_ctx->comp_data->func_type_count, + type_idx); + ftype_idx_const = I32_CONST(type_idx); + CHECK_LLVM_CONST(ftype_idx_const); func_type = comp_ctx->comp_data->func_types[type_idx]; - - param_cell_num = func_type->param_cell_num; - result_count = func_type->result_count; - wasm_ret_types = func_type->types + func_type->param_count; + func_param_count = func_type->param_count; + func_result_count = func_type->result_count; POP_I32(elem_idx); + if (comp_ctx->comp_data->import_table_count > 0) { + table_init_size = comp_ctx->comp_data->import_tables[0] + .table_init_size; + } + else if (comp_ctx->comp_data->table_count > 0) { + table_init_size = comp_ctx->comp_data->tables[0] + .table_init_size; + } + else { + aot_set_last_error("table index out of range"); + goto fail; + } + table_size_const = I32_CONST(table_init_size); + CHECK_LLVM_CONST(table_size_const); + + /* Check if (uint32)elem index >= table size */ + if (!(cmp_elem_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntUGE, + elem_idx, table_size_const, + "cmp_elem_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* Throw exception if elem index >= table size */ + if (!(check_elem_idx_succ = + LLVMAppendBasicBlockInContext(comp_ctx->context, + func_ctx->func, + "check_elem_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_elem_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNDEFINED_ELEMENT, + true, cmp_elem_idx, check_elem_idx_succ))) + goto fail; + + /* Load function index */ + if (!(table_elem = LLVMBuildInBoundsGEP(comp_ctx->builder, + func_ctx->table_base, + &elem_idx, 1, "table_elem"))) { + aot_set_last_error("llvm build add failed."); + goto fail; + } + + if (!(func_idx = LLVMBuildLoad(comp_ctx->builder, + table_elem, "func_idx"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + /* Check if func_idx == -1 */ + if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntEQ, + func_idx, I32_NEG_ONE, + "cmp_func_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* Throw exception if func_idx == -1 */ + if (!(check_func_idx_succ = + LLVMAppendBasicBlockInContext(comp_ctx->context, + func_ctx->func, + "check_func_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_func_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, EXCE_UNINITIALIZED_ELEMENT, + true, cmp_func_idx, check_func_idx_succ))) + goto fail; + + /* Load function type index */ + if (!(ftype_idx_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, + func_ctx->func_type_indexes, + &func_idx, 1, + "ftype_idx_ptr"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(ftype_idx = LLVMBuildLoad(comp_ctx->builder, ftype_idx_ptr, + "ftype_idx"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + /* Check if function type index not equal */ + if (!(cmp_ftype_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntNE, + ftype_idx, ftype_idx_const, + "cmp_ftype_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* Throw exception if ftype_idx != ftype_idx_const */ + if (!(check_ftype_idx_succ = + LLVMAppendBasicBlockInContext(comp_ctx->context, + func_ctx->func, + "check_ftype_idx_succ"))) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(check_ftype_idx_succ, + LLVMGetInsertBlock(comp_ctx->builder)); + + if (!(aot_emit_exception(comp_ctx, func_ctx, + EXCE_INVALID_FUNCTION_TYPE_INDEX, + true, cmp_ftype_idx, check_ftype_idx_succ))) + goto fail; + /* Initialize parameter types of the LLVM function */ - param_count = (int32)func_type->param_count; - total_size = sizeof(LLVMTypeRef) * (uint64)param_count; + total_param_count = 1 + func_param_count; + + /* Extra function results' addresses (except the first one) are + * appended to aot function parameters. */ + if (func_result_count > 1) + total_param_count += func_result_count - 1; + + total_size = sizeof(LLVMTypeRef) * (uint64)total_param_count; if (total_size >= UINT32_MAX || !(param_types = wasm_runtime_malloc((uint32)total_size))) { - aot_set_last_error("Allocate memory failed."); + aot_set_last_error("allocate memory failed."); goto fail; } - for (i = 0; i < param_count; i++) - param_types[i] = TO_LLVM_TYPE(func_type->types[i]); + /* Prepare param types */ + j = 0; + param_types[j++] = comp_ctx->exec_env_type; + for (i = 0; i < func_param_count; i++) + param_types[j++] = TO_LLVM_TYPE(func_type->types[i]); + + for (i = 1; i < func_result_count; i++, j++) { + param_types[j] = + TO_LLVM_TYPE(func_type->types[func_param_count + i]); + if (!(param_types[j] = LLVMPointerType(param_types[j], 0))) { + aot_set_last_error("llvm get pointer type failed."); + goto fail; + } + } + + /* Resolve return type of the LLVM function */ + if (func_result_count) { + wasm_ret_type = func_type->types[func_param_count]; + ret_type = TO_LLVM_TYPE(wasm_ret_type); + } + else { + wasm_ret_type = VALUE_TYPE_VOID; + ret_type = VOID_TYPE; + } /* Allocate memory for parameters */ - total_size = sizeof(LLVMValueRef) * (uint64)param_count; + total_size = sizeof(LLVMValueRef) * (uint64)total_param_count; if (total_size >= UINT32_MAX || !(param_values = wasm_runtime_malloc((uint32)total_size))) { - aot_set_last_error("Allocate memory failed."); + aot_set_last_error("allocate memory failed."); goto fail; } + /* First parameter is exec env */ + j = 0; + param_values[j++] = func_ctx->exec_env; + /* Pop parameters from stack */ - for (i = param_count - 1; i >= 0; i--) - POP(param_values[i], func_type->types[i]); + for (i = func_param_count - 1; (int32)i >= 0; i--) + POP(param_values[i + j], func_type->types[i]); + + /* Prepare extra parameters */ + ext_cell_num = 0; + for (i = 1; i < func_result_count; i++) { + ext_ret_offset = I32_CONST(ext_cell_num); + CHECK_LLVM_CONST(ext_ret_offset); + + snprintf(buf, sizeof(buf), "ext_ret%d_ptr", i - 1); + if (!(ext_ret_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, + func_ctx->argv_buf, + &ext_ret_offset, 1, buf))) { + aot_set_last_error("llvm build GEP failed."); + goto fail; + } + + ext_ret_ptr_type = param_types[func_param_count + i]; + snprintf(buf, sizeof(buf), "ext_ret%d_ptr_cast", i - 1); + if (!(ext_ret_ptr = LLVMBuildBitCast(comp_ctx->builder, + ext_ret_ptr, ext_ret_ptr_type, + buf))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + param_values[func_param_count + i] = ext_ret_ptr; + ext_cell_num += wasm_value_type_cell_num( + func_type->types[func_param_count + i]); + } + + if (ext_cell_num > 64) { + aot_set_last_error("prepare call-indirect arguments failed: " + "maximum 64 extra cell number supported."); + goto fail; + } + +#if WASM_ENABLE_THREAD_MGR != 0 + /* Insert suspend check point */ + if (comp_ctx->enable_thread_mgr) { + if (!check_suspend_flags(comp_ctx, func_ctx)) + goto fail; + } +#endif + + /* Add basic blocks */ + block_call_import = + LLVMAppendBasicBlockInContext(comp_ctx->context, func_ctx->func, + "call_import"); + block_call_non_import = + LLVMAppendBasicBlockInContext(comp_ctx->context, func_ctx->func, + "call_non_import"); + block_return = + LLVMAppendBasicBlockInContext(comp_ctx->context, func_ctx->func, + "func_return"); + if (!block_call_import || !block_call_non_import || !block_return) { + aot_set_last_error("llvm add basic block failed."); + goto fail; + } + + LLVMMoveBasicBlockAfter(block_call_import, + LLVMGetInsertBlock(comp_ctx->builder)); + LLVMMoveBasicBlockAfter(block_call_non_import, block_call_import); + LLVMMoveBasicBlockAfter(block_return, block_call_non_import); + + import_func_count = I32_CONST(comp_ctx->comp_data->import_func_count); + CHECK_LLVM_CONST(import_func_count); + + /* Check if func_idx < import_func_count */ + if (!(cmp_func_idx = LLVMBuildICmp(comp_ctx->builder, LLVMIntULT, + func_idx, import_func_count, + "cmp_func_idx"))) { + aot_set_last_error("llvm build icmp failed."); + goto fail; + } + + /* If func_idx < import_func_count, jump to call import block, + else jump to call non-import block */ + if (!LLVMBuildCondBr(comp_ctx->builder, cmp_func_idx, + block_call_import, block_call_non_import)) { + aot_set_last_error("llvm build cond br failed."); + goto fail; + } + + /* Add result phis for return block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return); + + if (func_result_count > 0) { + total_size = sizeof(LLVMValueRef) * (uint64)func_result_count; + if (total_size >= UINT32_MAX + || !(result_phis = wasm_runtime_malloc((uint32)total_size))) { + aot_set_last_error("allocate memory failed."); + goto fail; + } + memset(result_phis, 0, (uint32)total_size); + for (i = 0; i < func_result_count; i++) { + LLVMTypeRef tmp_type = + TO_LLVM_TYPE(func_type->types[func_param_count + i]); + if (!(result_phis[i] = LLVMBuildPhi(comp_ctx->builder, + tmp_type, "phi"))) { + aot_set_last_error("llvm build phi failed."); + goto fail; + } + } + } + + /* Translate call import block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_import); /* Allocate memory for result values */ - total_size = sizeof(LLVMValueRef) * (uint64)result_count; + total_size = sizeof(LLVMValueRef) * (uint64)func_result_count; if (total_size >= UINT32_MAX || !(value_rets = wasm_runtime_malloc((uint32)total_size))) { - aot_set_last_error("Allocate memory failed."); + aot_set_last_error("allocate memory failed."); goto fail; } memset(value_rets, 0, total_size); + param_cell_num = func_type->param_cell_num; + wasm_ret_types = func_type->types + func_type->param_count; + if (!call_aot_call_indirect_func(comp_ctx, func_ctx, func_type, ftype_idx, elem_idx, - param_types, param_values, - param_count, param_cell_num, - result_count, wasm_ret_types, + param_types + 1, param_values + 1, + func_param_count, param_cell_num, + func_result_count, wasm_ret_types, value_rets, &res)) goto fail; - for (i = 0; i < func_type->result_count; i++) - PUSH(value_rets[i], func_type->types[func_type->param_count + i]); - - /* Check whether there was exception thrown when executing the function */ + /* Check whether exception was thrown when executing the function */ if (!check_call_return(comp_ctx, func_ctx, res)) goto fail; + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + for (i = 0; i < func_result_count; i++) { + LLVMAddIncoming(result_phis[i], &value_rets[i], &block_curr, 1); + } + + if (!LLVMBuildBr(comp_ctx->builder, block_return)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Translate call non-import block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_call_non_import); + + if (comp_ctx->enable_bound_check + && !check_stack_boundary(comp_ctx, func_ctx, + param_cell_num + ext_cell_num + 1 + /* Reserve some local variables */ + + 16)) + goto fail; + + /* Load function pointer */ + if (!(func_ptr = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->func_ptrs, + &func_idx, 1, "func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(func_ptr = LLVMBuildLoad(comp_ctx->builder, func_ptr, "func_ptr"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(llvm_func_type = LLVMFunctionType(ret_type, param_types, + total_param_count, false)) + || !(llvm_func_ptr_type = LLVMPointerType(llvm_func_type, 0))) { + aot_set_last_error("llvm add function type failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, + func_ptr, llvm_func_ptr_type, + "indirect_func"))) { + aot_set_last_error("llvm build bit cast failed."); + goto fail; + } + + if (!(value_ret = LLVMBuildCall(comp_ctx->builder, func, + param_values, total_param_count, + func_result_count > 0 + ? "ret" : ""))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + /* Check whether exception was thrown when executing the function */ + if (!check_exception_thrown(comp_ctx, func_ctx)) + goto fail; + + if (func_result_count > 0) { + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + /* Push the first result to stack */ + LLVMAddIncoming(result_phis[0], &value_ret, &block_curr, 1); + + /* Load extra result from its address and push to stack */ + for (i = 1; i < func_result_count; i++) { + snprintf(buf, sizeof(buf), "ext_ret%d", i - 1); + if (!(ext_ret = LLVMBuildLoad(comp_ctx->builder, + param_values[func_param_count + i], + buf))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + LLVMAddIncoming(result_phis[i], &ext_ret, &block_curr, 1); + } + } + + if (!LLVMBuildBr(comp_ctx->builder, block_return)) { + aot_set_last_error("llvm build br failed."); + goto fail; + } + + /* Translate function return block */ + LLVMPositionBuilderAtEnd(comp_ctx->builder, block_return); + + for (i = 0; i < func_result_count; i++) { + PUSH(result_phis[i], func_type->types[func_param_count + i]); + } + ret = true; fail: - if (value_rets) - wasm_runtime_free(value_rets); if (param_values) wasm_runtime_free(param_values); if (param_types) wasm_runtime_free(param_types); + if (value_rets) + wasm_runtime_free(value_rets); + if (result_phis) + wasm_runtime_free(result_phis); return ret; } diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 2502fda4..26338dc6 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -465,9 +465,13 @@ create_memory_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, static bool create_table_base(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { + AOTCompData *comp_data = comp_ctx->comp_data; + uint64 module_inst_mem_inst_size = + (uint64)comp_data->memory_count * sizeof(AOTMemoryInstance); LLVMValueRef offset; offset = I32_CONST(offsetof(AOTModuleInstance, global_table_data.bytes) + + module_inst_mem_inst_size + comp_ctx->comp_data->global_data_size); func_ctx->table_base = LLVMBuildInBoundsGEP(comp_ctx->builder, func_ctx->aot_inst, @@ -544,6 +548,43 @@ create_func_type_indexes(AOTCompContext *comp_ctx, return true; } +static bool +create_func_ptrs(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) +{ + LLVMValueRef offset; + + offset = I32_CONST(offsetof(AOTModuleInstance, func_ptrs)); + func_ctx->func_ptrs = LLVMBuildInBoundsGEP(comp_ctx->builder, + func_ctx->aot_inst, + &offset, 1, "func_ptrs_offset"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build in bounds gep failed."); + return false; + } + func_ctx->func_ptrs = LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs, + comp_ctx->exec_env_type, "func_ptrs_tmp"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + func_ctx->func_ptrs = LLVMBuildLoad(comp_ctx->builder, func_ctx->func_ptrs, + "func_ptrs_ptr"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build load failed."); + return false; + } + + func_ctx->func_ptrs = LLVMBuildBitCast(comp_ctx->builder, func_ctx->func_ptrs, + comp_ctx->exec_env_type, "func_ptrs"); + if (!func_ctx->func_ptrs) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + return true; +} + /** * Create function compiler context */ @@ -746,6 +787,10 @@ aot_create_func_context(AOTCompData *comp_data, AOTCompContext *comp_ctx, if (!create_func_type_indexes(comp_ctx, func_ctx)) goto fail; + /* Load function pointers */ + if (!create_func_ptrs(comp_ctx, func_ctx)) + goto fail; + return func_ctx; fail: diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 30163a8b..599e664d 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -122,6 +122,7 @@ typedef struct AOTFuncContext { LLVMValueRef argv_buf; LLVMValueRef native_stack_bound; LLVMValueRef last_alloca; + LLVMValueRef func_ptrs; AOTMemInfo *mem_info; @@ -130,7 +131,6 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; - LLVMBasicBlockRef *exception_blocks; LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; LLVMValueRef exception_id_phi; diff --git a/core/iwasm/interpreter/wasm.h b/core/iwasm/interpreter/wasm.h index f6034d56..c8ab721d 100644 --- a/core/iwasm/interpreter/wasm.h +++ b/core/iwasm/interpreter/wasm.h @@ -517,6 +517,19 @@ wasm_type_equal(const WASMType *type1, const WASMType *type2) ? true : false; } +inline static uint32 +wasm_get_smallest_type_idx(WASMType **types, uint32 type_count, + uint32 cur_type_idx) +{ + uint32 i; + + for (i = 0; i < cur_type_idx; i++) { + if (wasm_type_equal(types[cur_type_idx], types[i])) + return i; + } + return cur_type_idx; +} + static inline uint32 block_type_get_param_types(BlockType *block_type, uint8 **p_param_types) diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 2815e63d..53810bad 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -380,48 +380,11 @@ popcount64(uint64 u) return ret; } -static uint64 -read_leb(const uint8 *buf, uint32 *p_offset, uint32 maxbits, bool sign) -{ - uint64 result = 0; - uint32 shift = 0; - uint32 bcnt = 0; - uint64 byte; - - while (true) { - byte = buf[*p_offset]; - *p_offset += 1; - result |= ((byte & 0x7f) << shift); - shift += 7; - if ((byte & 0x80) == 0) { - break; - } - bcnt += 1; - } - if (sign && (shift < maxbits) && (byte & 0x40)) { - /* Sign extend */ - result |= - ((uint64)1 << shift); - } - return result; -} - -#define read_leb_uint32(p, p_end, res) do { \ - uint8 _val = *p; \ - if (!(_val & 0x80)) { \ - res = _val; \ - p++; \ - break; \ - } \ - uint32 _off = 0; \ - res = (uint32)read_leb(p, &_off, 32, false); \ - p += _off; \ -} while (0) - #define read_uint32(p) (p += sizeof(uint32), *(uint32 *)(p - sizeof(uint32))) #define GET_LOCAL_INDEX_TYPE_AND_OFFSET() do { \ uint32 param_count = cur_func->param_count; \ - read_leb_uint32(frame_ip, frame_ip_end, local_idx); \ + local_idx = read_uint32(frame_ip); \ bh_assert(local_idx < param_count + cur_func->local_count); \ local_offset = cur_func->local_offsets[local_idx]; \ if (local_idx < param_count) \ diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 3db4c679..5315ca4a 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -791,6 +791,12 @@ load_function_import(const WASMModule *parent_module, WASMModule *sub_module, return false; } +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + declare_type_index = wasm_get_smallest_type_idx( + parent_module->types, parent_module->type_count, + declare_type_index); +#endif + declare_func_type = parent_module->types[declare_type_index]; if (wasm_runtime_is_host_module(sub_module_name)) { @@ -1724,6 +1730,11 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, return false; } +#if (WASM_ENABLE_WAMR_COMPILER != 0) || (WASM_ENABLE_JIT != 0) + type_index = wasm_get_smallest_type_idx( + module->types, module->type_count, type_index); +#endif + read_leb_uint32(p_code, buf_code_end, code_size); if (code_size == 0 || p_code + code_size > buf_code_end) { @@ -4312,10 +4323,6 @@ fail: LOG_OP("%d\t", value); \ } while (0) -#define emit_leb() do { \ - wasm_loader_emit_leb(loader_ctx, p_org, p); \ - } while (0) - static bool wasm_loader_ctx_reinit(WASMLoaderContext *ctx) { @@ -4402,21 +4409,6 @@ wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size) ctx->code_compiled_size -= size; } -static void -wasm_loader_emit_leb(WASMLoaderContext *ctx, uint8* start, uint8* end) -{ - if (ctx->p_code_compiled) { - bh_memcpy_s(ctx->p_code_compiled, - ctx->p_code_compiled_end - ctx->p_code_compiled, - start, end - start); - ctx->p_code_compiled += (end - start); - } - else { - ctx->code_compiled_size += (end - start); - } - -} - static bool preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode, uint32 local_index, uint32 local_type, bool *preserved, @@ -6497,8 +6489,7 @@ handle_op_block_and_loop: } } else { /* local index larger than 255, reserve leb */ - p_org ++; - emit_leb(); + emit_uint32(loader_ctx, local_idx); POP_OFFSET_TYPE(local_type); } #else @@ -6554,11 +6545,10 @@ handle_op_block_and_loop: } } else { /* local index larger than 255, reserve leb */ - p_org ++; - emit_leb(); + emit_uint32(loader_ctx, local_idx); } emit_operand(loader_ctx, *(loader_ctx->frame_offset - - wasm_value_type_cell_num(local_type))); + wasm_value_type_cell_num(local_type))); #else #if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) if (local_offset < 0x80) { diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index d082d46a..2dc38768 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -3062,10 +3062,6 @@ wasm_loader_pop_frame_csp(WASMLoaderContext *ctx, LOG_OP("%d\t", value); \ } while (0) -#define emit_leb() do { \ - wasm_loader_emit_leb(loader_ctx, p_org, p); \ - } while (0) - static bool wasm_loader_ctx_reinit(WASMLoaderContext *ctx) { @@ -3152,21 +3148,6 @@ wasm_loader_emit_backspace(WASMLoaderContext *ctx, uint32 size) ctx->code_compiled_size -= size; } -static void -wasm_loader_emit_leb(WASMLoaderContext *ctx, uint8* start, uint8* end) -{ - if (ctx->p_code_compiled) { - bh_memcpy_s(ctx->p_code_compiled, - ctx->p_code_compiled_end - ctx->p_code_compiled, - start, end - start); - ctx->p_code_compiled += (end - start); - } - else { - ctx->code_compiled_size += (end - start); - } - -} - static bool preserve_referenced_local(WASMLoaderContext *loader_ctx, uint8 opcode, uint32 local_index, uint32 local_type, bool *preserved, @@ -4970,8 +4951,7 @@ handle_op_block_and_loop: } } else { /* local index larger than 255, reserve leb */ - p_org ++; - emit_leb(); + emit_uint32(loader_ctx, local_idx); POP_OFFSET_TYPE(local_type); } #else @@ -5027,11 +5007,10 @@ handle_op_block_and_loop: } } else { /* local index larger than 255, reserve leb */ - p_org ++; - emit_leb(); + emit_uint32(loader_ctx, local_idx); } emit_operand(loader_ctx, *(loader_ctx->frame_offset - - wasm_value_type_cell_num(local_type))); + wasm_value_type_cell_num(local_type))); #else #if (WASM_ENABLE_WAMR_COMPILER == 0) && (WASM_ENABLE_JIT == 0) if (local_offset < 0x80) { diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 12d10f5d..6bf75c51 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1688,7 +1688,17 @@ wasm_module_malloc(WASMModuleInstance *module_inst, uint32 size, } if (!addr) { - wasm_set_exception(module_inst, "out of memory"); + if (memory->heap_handle + && mem_allocator_is_heap_corrupted(memory->heap_handle)) { + LOG_ERROR("Error: app heap is corrupted, if the wasm file " + "is compiled by wasi-sdk-12.0 or larger version, " + "please add -Wl,--export=malloc -Wl,--export=free " + " to export malloc and free functions."); + wasm_set_exception(module_inst, "app heap corrupted"); + } + else { + wasm_set_exception(module_inst, "out of memory"); + } return 0; } if (p_native_addr) diff --git a/core/shared/mem-alloc/ems/ems_gc.h b/core/shared/mem-alloc/ems/ems_gc.h index c712c908..a9db04c7 100644 --- a/core/shared/mem-alloc/ems/ems_gc.h +++ b/core/shared/mem-alloc/ems/ems_gc.h @@ -109,12 +109,14 @@ gc_migrate(gc_handle_t handle, char *pool_buf_new, gc_size_t pool_buf_size); /** - * Destroy lock of heap + * Check whether the heap is corrupted * - * @param handle the heap handle + * @param handle handle of the heap + * + * @return true if success, false otherwise */ -void -gc_destroy_lock(gc_handle_t handle); +bool +gc_is_heap_corrupted(gc_handle_t handle); /** * Get Heap Stats diff --git a/core/shared/mem-alloc/ems/ems_kfc.c b/core/shared/mem-alloc/ems/ems_kfc.c index ddfc5e4f..95664dc9 100644 --- a/core/shared/mem-alloc/ems/ems_kfc.c +++ b/core/shared/mem-alloc/ems/ems_kfc.c @@ -204,11 +204,12 @@ gc_migrate(gc_handle_t handle, return 0; } -void -gc_destroy_lock(gc_handle_t handle) +bool +gc_is_heap_corrupted(gc_handle_t handle) { - gc_heap_t *heap = (gc_heap_t *) handle; - os_mutex_destroy(&heap->lock); + gc_heap_t *heap = (gc_heap_t *)handle; + + return heap->is_heap_corrupted ? true : false; } #if BH_ENABLE_GC_VERIFY != 0 diff --git a/core/shared/mem-alloc/mem_alloc.c b/core/shared/mem-alloc/mem_alloc.c index f2304f8e..b460e6d1 100644 --- a/core/shared/mem-alloc/mem_alloc.c +++ b/core/shared/mem-alloc/mem_alloc.c @@ -63,10 +63,10 @@ mem_allocator_migrate(mem_allocator_t allocator, pool_buf_new, pool_buf_size); } -void -mem_allocator_destroy_lock(mem_allocator_t allocator) +bool +mem_allocator_is_heap_corrupted(mem_allocator_t allocator) { - gc_destroy_lock((gc_handle_t) allocator); + return gc_is_heap_corrupted((gc_handle_t) allocator); } #else /* else of DEFAULT_MEM_ALLOCATOR */ @@ -181,19 +181,5 @@ mem_allocator_migrate(mem_allocator_t allocator, (mem_allocator_tlsf *) allocator_old); } -int -mem_allocator_init_lock(mem_allocator_t allocator) -{ - mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; - return os_mutex_init(&allocator_tlsf->lock); -} - -void -mem_allocator_destroy_lock(mem_allocator_t allocator) -{ - mem_allocator_tlsf *allocator_tlsf = (mem_allocator_tlsf *)allocator; - os_mutex_destroy(&allocator_tlsf->lock); -} - #endif /* end of DEFAULT_MEM_ALLOCATOR */ diff --git a/core/shared/mem-alloc/mem_alloc.h b/core/shared/mem-alloc/mem_alloc.h index 21e97f5b..8224889e 100644 --- a/core/shared/mem-alloc/mem_alloc.h +++ b/core/shared/mem-alloc/mem_alloc.h @@ -42,8 +42,8 @@ int mem_allocator_migrate(mem_allocator_t allocator, char *pool_buf_new, uint32 pool_buf_size); -void -mem_allocator_destroy_lock(mem_allocator_t allocator); +bool +mem_allocator_is_heap_corrupted(mem_allocator_t allocator); #ifdef __cplusplus }