From ce3458da9945e33d3ef9c054e3012f751d32b7b2 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 30 Nov 2022 20:18:28 +0800 Subject: [PATCH] Refine AOT exception check when function return (#1752) Refine AOT exception check in the caller when returning from callee function, remove the exception check instructions when hw bound check is enabled to improve the performance: create guard page to trigger signal handler when exception occurs. --- core/iwasm/aot/aot_runtime.c | 85 +++++++++++++++------- core/iwasm/common/wasm_exec_env.c | 15 ++++ core/iwasm/common/wasm_exec_env.h | 5 +- core/iwasm/common/wasm_runtime_common.c | 17 +++++ core/iwasm/common/wasm_runtime_common.h | 6 ++ core/iwasm/compilation/aot_emit_function.c | 23 ++++-- core/iwasm/compilation/aot_llvm_extra.cpp | 11 +-- core/iwasm/interpreter/wasm_runtime.c | 51 ++++++++++--- 8 files changed, 163 insertions(+), 50 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 1c3f033c..1077c3af 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1500,7 +1500,11 @@ aot_set_exception(AOTModuleInstance *module_inst, const char *exception) void aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id) { - wasm_set_exception_with_id(module_inst, id); + if (id != EXCE_ALREADY_THROWN) + wasm_set_exception_with_id(module_inst, id); +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif } const char * @@ -1755,6 +1759,7 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, const char *signature; void *attachment; char buf[96]; + bool ret = false; bh_assert(func_idx < aot_module->import_func_count); @@ -1764,27 +1769,34 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, "failed to call unlinked import function (%s, %s)", import_func->module_name, import_func->func_name); aot_set_exception(module_inst, buf); - return false; + goto fail; } attachment = import_func->attachment; if (import_func->call_conv_wasm_c_api) { - return wasm_runtime_invoke_c_api_native( + ret = wasm_runtime_invoke_c_api_native( (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, argv, import_func->wasm_c_api_with_env, attachment); } else if (!import_func->call_conv_raw) { signature = import_func->signature; - return wasm_runtime_invoke_native(exec_env, func_ptr, func_type, - signature, attachment, argv, argc, - argv); + ret = + wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv); } else { signature = import_func->signature; - return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, - signature, attachment, argv, argc, - argv); + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; } bool @@ -1811,7 +1823,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, if ((uint8 *)&module_inst < exec_env->native_stack_boundary) { aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW); - return false; + goto fail; } tbl_inst = module_inst->tables[tbl_idx]; @@ -1819,13 +1831,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, if (table_elem_idx >= tbl_inst->cur_size) { aot_set_exception_with_id(module_inst, EXCE_UNDEFINED_ELEMENT); - return false; + goto fail; } func_idx = tbl_inst->elems[table_elem_idx]; if (func_idx == NULL_REF) { aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT); - return false; + goto fail; } func_type_idx = func_type_indexes[func_idx]; @@ -1843,7 +1855,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, "failed to call unlinked import function (%s, %s)", import_func->module_name, import_func->func_name); aot_set_exception(module_inst, buf); - return false; + goto fail; } if (func_idx < aot_module->import_func_count) { @@ -1852,9 +1864,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, signature = import_func->signature; if (import_func->call_conv_raw) { attachment = import_func->attachment; - return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, - signature, attachment, argv, - argc, argv); + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, + argc, argv); + if (!ret) + goto fail; + + return true; } } @@ -1878,7 +1894,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, && !(argv1 = runtime_malloc(size, module_inst->cur_exception, sizeof(module_inst->cur_exception)))) { aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY); - return false; + goto fail; } /* Copy original arguments */ @@ -1897,12 +1913,10 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, ret = invoke_native_internal(exec_env, func_ptr, func_type, signature, attachment, argv1, argc, argv); - if (!ret || aot_get_exception(module_inst)) { + if (!ret) { if (argv1 != argv1_buf) wasm_runtime_free(argv1); - if (clear_wasi_proc_exit_exception(module_inst)) - return true; - return false; + goto fail; } /* Get extra result values */ @@ -1941,10 +1955,20 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, else { ret = invoke_native_internal(exec_env, func_ptr, func_type, signature, attachment, argv, argc, argv); - if (clear_wasi_proc_exit_exception(module_inst)) - return true; - return ret; + if (!ret) + goto fail; + + return true; } + +fail: + if (clear_wasi_proc_exit_exception(module_inst)) + return true; + +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif + return false; } bool @@ -1952,8 +1976,17 @@ aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, uint32 app_buf_addr, uint32 app_buf_size, void **p_native_addr) { - return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, - app_buf_size, p_native_addr); + bool ret; + + ret = wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, + app_buf_size, p_native_addr); + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + + return ret; } void * diff --git a/core/iwasm/common/wasm_exec_env.c b/core/iwasm/common/wasm_exec_env.c index cd8b5708..515a60b8 100644 --- a/core/iwasm/common/wasm_exec_env.c +++ b/core/iwasm/common/wasm_exec_env.c @@ -56,6 +56,12 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, #endif #endif +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!(exec_env->exce_check_guard_page = + os_mmap(NULL, os_getpagesize(), MMAP_PROT_NONE, MMAP_MAP_NONE))) + goto fail5; +#endif + exec_env->module_inst = module_inst; exec_env->wasm_stack_size = stack_size; exec_env->wasm_stack.s.top_boundary = @@ -76,6 +82,12 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst, return exec_env; +#ifdef OS_ENABLE_HW_BOUND_CHECK +fail5: +#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0 + wasm_cluster_destroy_exenv_status(exec_env->current_status); +#endif +#endif #if WASM_ENABLE_THREAD_MGR != 0 #if WASM_ENABLE_DEBUG_INTERP != 0 fail4: @@ -96,6 +108,9 @@ fail1: void wasm_exec_env_destroy_internal(WASMExecEnv *exec_env) { +#ifdef OS_ENABLE_HW_BOUND_CHECK + os_munmap(exec_env->exce_check_guard_page, os_getpagesize()); +#endif #if WASM_ENABLE_THREAD_MGR != 0 os_mutex_destroy(&exec_env->wait_lock); os_cond_destroy(&exec_env->wait_cond); diff --git a/core/iwasm/common/wasm_exec_env.h b/core/iwasm/common/wasm_exec_env.h index d4d7ce00..39829207 100644 --- a/core/iwasm/common/wasm_exec_env.h +++ b/core/iwasm/common/wasm_exec_env.h @@ -137,6 +137,8 @@ typedef struct WASMExecEnv { #ifdef OS_ENABLE_HW_BOUND_CHECK WASMJmpBuf *jmpbuf_stack_top; + /* One guard page for the exception check */ + uint8 *exce_check_guard_page; #endif #if WASM_ENABLE_MEMORY_PROFILING != 0 @@ -199,7 +201,8 @@ wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size) the outs area contains const cells, its size may be larger than current frame size, we should check again before putting the function arguments into the outs area. */ - if (addr + size * 2 > exec_env->wasm_stack.s.top_boundary) { + if (size * 2 + > (uint32)(uintptr_t)(exec_env->wasm_stack.s.top_boundary - addr)) { /* WASM stack overflow. */ return NULL; } diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index fbcdfcd0..17cf9da3 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -185,6 +185,12 @@ runtime_signal_handler(void *sig_addr) os_longjmp(jmpbuf_node->jmpbuf, 1); } #endif + else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr + && (uint8 *)sig_addr + < exec_env_tls->exce_check_guard_page + page_size) { + bh_assert(wasm_get_exception(module_inst)); + os_longjmp(jmpbuf_node->jmpbuf, 1); + } } } #else @@ -1435,6 +1441,17 @@ wasm_runtime_get_user_data(WASMExecEnv *exec_env) return exec_env->user_data; } +#ifdef OS_ENABLE_HW_BOUND_CHECK +void +wasm_runtime_access_exce_check_guard_page() +{ + if (exec_env_tls && exec_env_tls->handle == os_self_thread()) { + uint32 page_size = os_getpagesize(); + memset(exec_env_tls->exce_check_guard_page, 0, page_size); + } +} +#endif + WASMType * wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function, uint32 module_type) diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index cc311ba7..3e7f3725 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -554,6 +554,12 @@ wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data); WASM_RUNTIME_API_EXTERN void * wasm_runtime_get_user_data(WASMExecEnv *exec_env); +#ifdef OS_ENABLE_HW_BOUND_CHECK +/* Access exception check guard page to trigger the signal handler */ +void +wasm_runtime_access_exce_check_guard_page(); +#endif + /* See wasm_export.h for description */ WASM_RUNTIME_API_EXTERN bool wasm_runtime_call_wasm(WASMExecEnv *exec_env, diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index cbcdebff..fb2fca56 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -35,7 +35,14 @@ create_func_return_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) /* Create return IR */ LLVMPositionBuilderAtEnd(comp_ctx->builder, func_ctx->func_return_block); - if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) { + if (!comp_ctx->enable_bound_check) { + if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ALREADY_THROWN, + false, NULL, NULL)) { + return false; + } + } + else if (!aot_build_zero_function_ret(comp_ctx, func_ctx, + aot_func_type)) { return false; } } @@ -494,7 +501,8 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Check whether exception was thrown when executing the function */ - if (!check_call_return(comp_ctx, func_ctx, res)) { + if (comp_ctx->enable_bound_check + && !check_call_return(comp_ctx, func_ctx, res)) { return false; } @@ -707,7 +715,8 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; /* Check whether there was exception thrown when executing the function */ - if (!check_call_return(comp_ctx, func_ctx, res)) + if (comp_ctx->enable_bound_check + && !check_call_return(comp_ctx, func_ctx, res)) goto fail; } else { /* call native func directly */ @@ -823,7 +832,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Check whether there was exception thrown when executing the function */ - if (!tail_call && !recursive_call + if (!tail_call && !recursive_call && comp_ctx->enable_bound_check && !check_exception_thrown(comp_ctx, func_ctx)) goto fail; } @@ -1395,7 +1404,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; /* Check whether exception was thrown when executing the function */ - if (!check_call_return(comp_ctx, func_ctx, res)) + if (comp_ctx->enable_bound_check + && !check_call_return(comp_ctx, func_ctx, res)) goto fail; block_curr = LLVMGetInsertBlock(comp_ctx->builder); @@ -1454,7 +1464,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Check whether exception was thrown when executing the function */ - if (!check_exception_thrown(comp_ctx, func_ctx)) + if (comp_ctx->enable_bound_check + && !check_exception_thrown(comp_ctx, func_ctx)) goto fail; if (func_result_count > 0) { diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp index 52e219b2..9b77f5e6 100644 --- a/core/iwasm/compilation/aot_llvm_extra.cpp +++ b/core/iwasm/compilation/aot_llvm_extra.cpp @@ -344,11 +344,12 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module) if (!disable_llvm_lto) { /* Apply LTO for AOT mode */ -#if LLVM_VERSION_MAJOR < 14 - MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL)); -#else - MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL)); -#endif + if (comp_ctx->comp_data->func_count >= 10) + /* Adds the pre-link optimizations if the func count + is large enough */ + MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL)); + else + MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL)); } else { MPM.addPass(PB.buildPerModuleDefaultPipeline(OL)); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index f702a0e8..8e8f96b5 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2781,7 +2781,11 @@ wasm_interp_dump_call_stack(struct WASMExecEnv *exec_env, bool print, char *buf, void jit_set_exception_with_id(WASMModuleInstance *module_inst, uint32 id) { - wasm_set_exception_with_id(module_inst, id); + if (id != EXCE_ALREADY_THROWN) + wasm_set_exception_with_id(module_inst, id); +#ifdef OS_ENABLE_HW_BOUND_CHECK + wasm_runtime_access_exce_check_guard_page(); +#endif } bool @@ -2789,8 +2793,15 @@ jit_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, uint32 app_buf_addr, uint32 app_buf_size, void **p_native_addr) { - return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr, - app_buf_size, p_native_addr); + bool ret = wasm_check_app_addr_and_convert( + module_inst, is_str, app_buf_addr, app_buf_size, p_native_addr); + +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + + return ret; } #endif /* end of WASM_ENABLE_FAST_JIT != 0 || WASM_ENABLE_JIT != 0 \ || WASM_ENABLE_WAMR_COMPILER != 0 */ @@ -2811,12 +2822,20 @@ bool llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, uint32 argc, uint32 *argv) { + bool ret; + #if WASM_ENABLE_JIT != 0 if (Wasm_Module_AoT == exec_env->module_inst->module_type) { return aot_call_indirect(exec_env, tbl_idx, elem_idx, argc, argv); } #endif - return call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); + + ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; } bool @@ -2833,6 +2852,7 @@ llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, const char *signature; void *attachment; char buf[96]; + bool ret = false; #if WASM_ENABLE_JIT != 0 if (Wasm_Module_AoT == exec_env->module_inst->module_type) { @@ -2855,27 +2875,34 @@ llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, "failed to call unlinked import function (%s, %s)", import_func->module_name, import_func->field_name); wasm_set_exception(module_inst, buf); - return false; + goto fail; } attachment = import_func->attachment; if (import_func->call_conv_wasm_c_api) { - return wasm_runtime_invoke_c_api_native( + ret = wasm_runtime_invoke_c_api_native( (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc, argv, import_func->wasm_c_api_with_env, attachment); } else if (!import_func->call_conv_raw) { signature = import_func->signature; - return wasm_runtime_invoke_native(exec_env, func_ptr, func_type, - signature, attachment, argv, argc, - argv); + ret = + wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature, + attachment, argv, argc, argv); } else { signature = import_func->signature; - return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, - signature, attachment, argv, argc, - argv); + ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type, + signature, attachment, argv, argc, + argv); } + +fail: +#ifdef OS_ENABLE_HW_BOUND_CHECK + if (!ret) + wasm_runtime_access_exce_check_guard_page(); +#endif + return ret; } #if WASM_ENABLE_BULK_MEMORY != 0