From 25fc006c3359e0788b42bc9a11923f8ffbe29577 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 23 Feb 2022 14:58:32 +0800 Subject: [PATCH] Refine call native function from AOT code (#1015) When calling native function from AOT code, current implementation is to return back to runtime to call aot_invoke_native, which calls wasm_runtime_invoke_native and the latter calls assembly code. We did it before as there may be pointer and string arguments to check and convert if the native function's registered signature has character '*' and '$'. As the built-in native function's signatures can be gotten in compilation time, we check the pointer/string arguments and convert them into native address in AOT code, and then invoke the native function directly, so as to improve performance. --- core/iwasm/aot/aot_reloc.h | 1 + core/iwasm/aot/aot_runtime.c | 50 +++++ core/iwasm/aot/aot_runtime.h | 5 + core/iwasm/compilation/aot_emit_function.c | 221 +++++++++++++++++++-- wamr-compiler/CMakeLists.txt | 4 + 5 files changed, 267 insertions(+), 14 deletions(-) diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index c0ca507e..471bf9c0 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -113,6 +113,7 @@ typedef struct { REG_SYM(aot_call_indirect), \ REG_SYM(aot_enlarge_memory), \ REG_SYM(aot_set_exception), \ + REG_SYM(aot_check_app_addr_and_convert),\ { "memset", (void*)aot_memset }, \ { "memmove", (void*)aot_memmove }, \ { "memcpy", (void*)aot_memmove }, \ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index baa5ce7b..e4b19a2d 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -2431,6 +2431,56 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, } } +/** + * Check whether the app address and the buf is inside the linear memory, + * and convert the app address into native address + */ +bool +aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr) +{ + AOTMemoryInstance *memory_inst = aot_get_default_memory(module_inst); + uint8 *native_addr; + + if (!memory_inst) { + goto fail; + } + + native_addr = (uint8 *)memory_inst->memory_data.ptr + app_buf_addr; + + /* No need to check the app_offset and buf_size if memory access + boundary check with hardware trap is enabled */ +#ifndef OS_ENABLE_HW_BOUND_CHECK + if (app_buf_addr >= memory_inst->memory_data_size) { + goto fail; + } + + if (!is_str) { + if (app_buf_size > memory_inst->memory_data_size - app_buf_addr) { + goto fail; + } + } + else { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)memory_inst->memory_data_end.ptr; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) + goto fail; + } +#endif + + *p_native_addr = (void *)native_addr; + return true; +fail: + aot_set_exception(module_inst, "out of bounds memory access"); + return false; +} + void * aot_memmove(void *dest, const void *src, size_t n) { diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 8046880a..5f1c42e7 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -647,6 +647,11 @@ bool aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx, uint32 argc, uint32 *argv); +bool +aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str, + uint32 app_buf_addr, uint32 app_buf_size, + void **p_native_addr); + uint32 aot_get_plt_table_size(); diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index c5f1379c..0510172f 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -518,6 +518,111 @@ check_stack_boundary(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +/** + * Check whether the app address and its buffer are inside the linear memory, + * if no, throw exception + */ +static bool +check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + bool is_str_arg, LLVMValueRef app_addr, + LLVMValueRef buf_size, + LLVMValueRef *p_native_addr_converted) +{ + LLVMTypeRef func_type, func_ptr_type, func_param_types[5]; + LLVMValueRef func, func_param_values[5], res, native_addr_ptr; + char *func_name = "aot_check_app_addr_and_convert"; + + /* prepare function type of aot_check_app_addr_and_convert */ + func_param_types[0] = comp_ctx->aot_inst_type; /* module_inst */ + func_param_types[1] = INT8_TYPE; /* is_str_arg */ + func_param_types[2] = I32_TYPE; /* app_offset */ + func_param_types[3] = I32_TYPE; /* buf_size */ + func_param_types[4] = + comp_ctx->basic_types.int8_pptr_type; /* p_native_addr */ + if (!(func_type = + LLVMFunctionType(INT8_TYPE, func_param_types, 5, false))) { + aot_set_last_error("llvm add function type failed."); + return false; + } + + /* prepare function pointer */ + if (comp_ctx->is_jit_mode) { + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + + /* JIT mode, call the function directly */ + if (!(func = + I64_CONST((uint64)(uintptr_t)aot_check_app_addr_and_convert)) + || !(func = LLVMConstIntToPtr(func, func_ptr_type))) { + aot_set_last_error("create LLVM value failed."); + return false; + } + } + else if (comp_ctx->is_indirect_mode) { + int32 func_index; + if (!(func_ptr_type = LLVMPointerType(func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + return false; + } + func_index = aot_get_native_symbol_index(comp_ctx, func_name); + if (func_index < 0) { + return false; + } + if (!(func = aot_get_func_from_table(comp_ctx, func_ctx->native_symbol, + func_ptr_type, func_index))) { + return false; + } + } + else { + if (!(func = LLVMGetNamedFunction(func_ctx->module, func_name)) + && !(func = + LLVMAddFunction(func_ctx->module, func_name, func_type))) { + aot_set_last_error("add LLVM function failed."); + return false; + } + } + + if (!(native_addr_ptr = LLVMBuildBitCast( + comp_ctx->builder, func_ctx->argv_buf, + comp_ctx->basic_types.int8_pptr_type, "p_native_addr"))) { + aot_set_last_error("llvm build bit cast failed."); + return false; + } + + func_param_values[0] = func_ctx->aot_inst; + func_param_values[1] = I8_CONST(is_str_arg); + func_param_values[2] = app_addr; + func_param_values[3] = buf_size; + func_param_values[4] = native_addr_ptr; + + if (!func_param_values[1]) { + aot_set_last_error("llvm create const failed."); + return false; + } + + /* call aot_check_app_addr_and_convert() function */ + if (!(res = LLVMBuildCall(comp_ctx->builder, func, func_param_values, 5, + "res"))) { + aot_set_last_error("llvm build call failed."); + return false; + } + + /* Check whether exception was thrown when executing the function */ + if (!check_call_return(comp_ctx, func_ctx, res)) { + return false; + } + + if (!(*p_native_addr_converted = LLVMBuildLoad( + comp_ctx->builder, native_addr_ptr, "native_addr"))) { + aot_set_last_error("llvm build load failed."); + return false; + } + + return true; +} + bool aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 func_idx, bool tail_call) @@ -539,6 +644,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, uint32 callee_cell_num; uint8 wasm_ret_type; uint8 *ext_ret_types = NULL; + const char *signature = NULL; bool ret = false; char buf[32]; @@ -557,11 +663,14 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Get function type */ - if (func_idx < import_func_count) + if (func_idx < import_func_count) { func_type = import_funcs[func_idx].func_type; - else + signature = import_funcs[func_idx].signature; + } + else { func_type = func_ctxes[func_idx - import_func_count]->aot_func->func_type; + } /* Get param cell number */ param_cell_num = func_type->param_cell_num; @@ -659,8 +768,41 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, j = 0; param_types[j++] = comp_ctx->exec_env_type; - for (i = 0; i < param_count; i++) - param_types[j++] = TO_LLVM_TYPE(func_type->types[i]); + for (i = 0; i < param_count; i++, j++) { + param_types[j] = TO_LLVM_TYPE(func_type->types[i]); + + /* If the signature can be gotten, e.g. the signature of the builtin + native libraries, just check the app offset and buf size, and + then convert app offset to native addr and call the native func + directly, no need to call aot_invoke_native to call it */ + if (signature) { + LLVMValueRef native_addr, native_addr_size; + if (signature[i + 1] == '*' || signature[i + 1] == '$') { + param_types[j] = INT8_PTR_TYPE; + } + if (signature[i + 1] == '*') { + if (signature[i + 2] == '~') + native_addr_size = param_values[i + 2]; + else + native_addr_size = I32_ONE; + if (!check_app_addr_and_convert( + comp_ctx, func_ctx, false, param_values[j], + native_addr_size, &native_addr)) { + goto fail; + } + param_values[j] = native_addr; + } + else if (signature[i + 1] == '$') { + native_addr_size = I32_ZERO; + if (!check_app_addr_and_convert( + comp_ctx, func_ctx, true, param_values[j], + native_addr_size, &native_addr)) { + goto fail; + } + param_values[j] = native_addr; + } + } + } if (func_type->result_count) { wasm_ret_type = func_type->types[func_type->param_count]; @@ -671,17 +813,68 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ret_type = VOID_TYPE; } - /* call aot_invoke_native() */ - if (!call_aot_invoke_native_func( - comp_ctx, func_ctx, import_func_idx, func_type, param_types + 1, - param_values + 1, param_count, param_cell_num, ret_type, - wasm_ret_type, &value_ret, &res)) - goto fail; + if (!signature) { + /* call aot_invoke_native() */ + if (!call_aot_invoke_native_func( + comp_ctx, func_ctx, import_func_idx, func_type, + param_types + 1, param_values + 1, param_count, + param_cell_num, ret_type, wasm_ret_type, &value_ret, &res)) + goto fail; + /* Check whether there was exception thrown when executing + the function */ + if (!check_call_return(comp_ctx, func_ctx, res)) + goto fail; + } + else { /* call native func directly */ + LLVMTypeRef native_func_type, func_ptr_type; + LLVMValueRef func_ptr; - /* Check whether there was exception thrown when executing - the function */ - if (!check_call_return(comp_ctx, func_ctx, res)) - goto fail; + if (!(native_func_type = LLVMFunctionType( + ret_type, param_types, param_count + 1, false))) { + aot_set_last_error("llvm add function type failed."); + goto fail; + } + + if (!(func_ptr_type = LLVMPointerType(native_func_type, 0))) { + aot_set_last_error("create LLVM function type failed."); + goto fail; + } + + /* Load function pointer */ + if (!(func_ptr = LLVMBuildInBoundsGEP( + comp_ctx->builder, func_ctx->func_ptrs, &import_func_idx, + 1, "native_func_ptr_tmp"))) { + aot_set_last_error("llvm build inbounds gep failed."); + goto fail; + } + + if (!(func_ptr = LLVMBuildLoad(comp_ctx->builder, func_ptr, + "native_func_ptr"))) { + aot_set_last_error("llvm build load failed."); + goto fail; + } + + if (!(func = LLVMBuildBitCast(comp_ctx->builder, func_ptr, + func_ptr_type, "native_func"))) { + aot_set_last_error("llvm bit cast failed."); + goto fail; + } + + /* Call the function */ + if (!(value_ret = LLVMBuildCall( + comp_ctx->builder, func, param_values, + (uint32)param_count + 1 + ext_ret_count, + (func_type->result_count > 0 ? "call" : "")))) { + aot_set_last_error("LLVM build call failed."); + goto fail; + } + + /* Check whether there was exception thrown when executing + the function */ + if (!check_exception_thrown(comp_ctx, func_ctx)) { + goto fail; + } + } } else { bool recursive_call = diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index e6f64d82..fa11192f 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -181,6 +181,8 @@ include (${SHARED_DIR}/utils/shared_utils.cmake) include (${SHARED_DIR}/utils/uncommon/shared_uncommon.cmake) include (${IWASM_DIR}/libraries/thread-mgr/thread_mgr.cmake) include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) +include (${IWASM_DIR}/libraries/libc-wasi/libc_wasi.cmake) +include (${IWASM_DIR}/libraries/lib-pthread/lib_pthread.cmake) include (${IWASM_DIR}/common/iwasm_common.cmake) include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) include (${IWASM_DIR}/aot/iwasm_aot.cmake) @@ -239,6 +241,8 @@ add_library (vmlib ${UNCOMMON_SHARED_SOURCE} ${THREAD_MGR_SOURCE} ${LIBC_BUILTIN_SOURCE} + ${LIBC_WASI_SOURCE} + ${LIB_PTHREAD_SOURCE} ${IWASM_COMMON_SOURCE} ${IWASM_INTERP_SOURCE} ${IWASM_AOT_SOURCE})