From 81fbfbfcc0ca6ee88ffa9b11d8007695ad61fb51 Mon Sep 17 00:00:00 2001 From: Cengizhan Pasaoglu Date: Tue, 8 Aug 2023 03:52:25 +0300 Subject: [PATCH 01/34] Remove unnecessary and extra zero length check in mem functions' macro (#2428) In macro bh_memcpy_s, bh_memcy_wa and bh_memmove_s, no need to do extra check for length is zero or not because it was already done inside of the functions called. --- core/shared/utils/bh_common.c | 4 ++++ core/shared/utils/bh_common.h | 30 +++++++++++++++--------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/core/shared/utils/bh_common.c b/core/shared/utils/bh_common.c index aeeab26b..7fe123c9 100644 --- a/core/shared/utils/bh_common.c +++ b/core/shared/utils/bh_common.c @@ -31,6 +31,10 @@ b_memcpy_wa(void *s1, unsigned int s1max, const void *s2, unsigned int n) unsigned int *p; char *ps; + if (n == 0) { + return 0; + } + if (pa > src) { pa -= 4; } diff --git a/core/shared/utils/bh_common.h b/core/shared/utils/bh_common.h index edb962eb..adae722b 100644 --- a/core/shared/utils/bh_common.h +++ b/core/shared/utils/bh_common.h @@ -12,25 +12,25 @@ extern "C" { #endif -#define bh_memcpy_s(dest, dlen, src, slen) \ - do { \ - int _ret = slen == 0 ? 0 : b_memcpy_s(dest, dlen, src, slen); \ - (void)_ret; \ - bh_assert(_ret == 0); \ +#define bh_memcpy_s(dest, dlen, src, slen) \ + do { \ + int _ret = b_memcpy_s(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ } while (0) -#define bh_memcpy_wa(dest, dlen, src, slen) \ - do { \ - int _ret = slen == 0 ? 0 : b_memcpy_wa(dest, dlen, src, slen); \ - (void)_ret; \ - bh_assert(_ret == 0); \ +#define bh_memcpy_wa(dest, dlen, src, slen) \ + do { \ + int _ret = b_memcpy_wa(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ } while (0) -#define bh_memmove_s(dest, dlen, src, slen) \ - do { \ - int _ret = slen == 0 ? 0 : b_memmove_s(dest, dlen, src, slen); \ - (void)_ret; \ - bh_assert(_ret == 0); \ +#define bh_memmove_s(dest, dlen, src, slen) \ + do { \ + int _ret = b_memmove_s(dest, dlen, src, slen); \ + (void)_ret; \ + bh_assert(_ret == 0); \ } while (0) #define bh_strcat_s(dest, dlen, src) \ From 51714c41c0c9210ecd0414849ea21d1d1dfcb937 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 8 Aug 2023 10:35:29 +0900 Subject: [PATCH 02/34] Introduce WASMModuleInstanceExtraCommon (#2429) Move the common parts of WASMModuleInstanceExtra and AOTModuleInstanceExtra into the new structure. --- core/iwasm/aot/aot_runtime.c | 10 +++++----- core/iwasm/aot/aot_runtime.h | 6 +----- core/iwasm/common/wasm_c_api.c | 4 ++-- core/iwasm/common/wasm_runtime_common.c | 8 ++++---- core/iwasm/interpreter/wasm_interp_classic.c | 5 +++-- core/iwasm/interpreter/wasm_interp_fast.c | 5 +++-- core/iwasm/interpreter/wasm_runtime.c | 9 +++++---- core/iwasm/interpreter/wasm_runtime.h | 16 +++++++++++----- core/iwasm/libraries/thread-mgr/thread_manager.c | 10 +++++----- 9 files changed, 39 insertions(+), 34 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 3e6371f4..94c66e67 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1271,9 +1271,9 @@ aot_deinstantiate(AOTModuleInstance *module_inst, bool is_sub_inst) if (module_inst->func_type_indexes) wasm_runtime_free(module_inst->func_type_indexes); - if (((AOTModuleInstanceExtra *)module_inst->e)->c_api_func_imports) - wasm_runtime_free( - ((AOTModuleInstanceExtra *)module_inst->e)->c_api_func_imports); + if (((AOTModuleInstanceExtra *)module_inst->e)->common.c_api_func_imports) + wasm_runtime_free(((AOTModuleInstanceExtra *)module_inst->e) + ->common.c_api_func_imports); if (!is_sub_inst) { #if WASM_ENABLE_LIBC_WASI != 0 @@ -1925,8 +1925,8 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, AOTModuleInstanceExtra *module_inst_extra = (AOTModuleInstanceExtra *)module_inst->e; CApiFuncImport *c_api_func_import = - module_inst_extra->c_api_func_imports - ? module_inst_extra->c_api_func_imports + func_idx + module_inst_extra->common.c_api_func_imports + ? module_inst_extra->common.c_api_func_imports + func_idx : NULL; uint32 *func_type_indexes = module_inst->func_type_indexes; uint32 func_type_idx = func_type_indexes[func_idx]; diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 2dea832b..895823af 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -89,11 +89,7 @@ typedef struct AOTFunctionInstance { typedef struct AOTModuleInstanceExtra { DefPointer(const uint32 *, stack_sizes); - CApiFuncImport *c_api_func_imports; -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 - /* Disable bounds checks or not */ - bool disable_bounds_checks; -#endif + WASMModuleInstanceExtraCommon common; } AOTModuleInstanceExtra; #if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index 7b8cf477..a595a626 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -4858,7 +4858,7 @@ wasm_instance_new_with_args(wasm_store_t *store, const wasm_module_t *module, if (instance->inst_comm_rt->module_type == Wasm_Module_Bytecode) { WASMModuleInstanceExtra *e = ((WASMModuleInstance *)instance->inst_comm_rt)->e; - p_func_imports = &(e->c_api_func_imports); + p_func_imports = &(e->common.c_api_func_imports); import_func_count = MODULE_INTERP(module)->import_function_count; } #endif @@ -4868,7 +4868,7 @@ wasm_instance_new_with_args(wasm_store_t *store, const wasm_module_t *module, (AOTModuleInstanceExtra *)((AOTModuleInstance *) instance->inst_comm_rt) ->e; - p_func_imports = &(e->c_api_func_imports); + p_func_imports = &(e->common.c_api_func_imports); import_func_count = MODULE_AOT(module)->import_func_count; } #endif diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index 8e793379..a39613a3 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -2491,14 +2491,14 @@ wasm_runtime_set_bounds_checks(WASMModuleInstanceCommon *module_inst, #if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { ((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e) - ->disable_bounds_checks = enable ? false : true; + ->common.disable_bounds_checks = enable ? false : true; } #endif #if WASM_ENABLE_AOT != 0 if (module_inst->module_type == Wasm_Module_AoT) { ((AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e) - ->disable_bounds_checks = enable ? false : true; + ->common.disable_bounds_checks = enable ? false : true; } #endif } @@ -2511,7 +2511,7 @@ wasm_runtime_is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_Bytecode) { return !((WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst) ->e) - ->disable_bounds_checks; + ->common.disable_bounds_checks; } #endif @@ -2519,7 +2519,7 @@ wasm_runtime_is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst) if (module_inst->module_type == Wasm_Module_AoT) { return !((AOTModuleInstanceExtra *)((WASMModuleInstance *)module_inst) ->e) - ->disable_bounds_checks; + ->common.disable_bounds_checks; } #endif diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index 94990095..2a48c1bd 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -905,8 +905,9 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, if (!func_import->call_conv_wasm_c_api) { native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; } - else if (module_inst->e->c_api_func_imports) { - c_api_func_import = module_inst->e->c_api_func_imports + cur_func_index; + else if (module_inst->e->common.c_api_func_imports) { + c_api_func_import = + module_inst->e->common.c_api_func_imports + cur_func_index; native_func_pointer = c_api_func_import->func_ptr_linked; } diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 729d4c60..229a2151 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -938,8 +938,9 @@ wasm_interp_call_func_native(WASMModuleInstance *module_inst, if (!func_import->call_conv_wasm_c_api) { native_func_pointer = module_inst->import_func_ptrs[cur_func_index]; } - else if (module_inst->e->c_api_func_imports) { - c_api_func_import = module_inst->e->c_api_func_imports + cur_func_index; + else if (module_inst->e->common.c_api_func_imports) { + c_api_func_import = + module_inst->e->common.c_api_func_imports + cur_func_index; native_func_pointer = c_api_func_import->func_ptr_linked; } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index eef2e368..0fdef512 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2227,8 +2227,8 @@ wasm_deinstantiate(WASMModuleInstance *module_inst, bool is_sub_inst) } #endif - if (module_inst->e->c_api_func_imports) - wasm_runtime_free(module_inst->e->c_api_func_imports); + if (module_inst->e->common.c_api_func_imports) + wasm_runtime_free(module_inst->e->common.c_api_func_imports); if (!is_sub_inst) { #if WASM_ENABLE_LIBC_WASI != 0 @@ -3141,8 +3141,9 @@ llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, import_func = &module->import_functions[func_idx].u.function; if (import_func->call_conv_wasm_c_api) { - if (module_inst->e->c_api_func_imports) { - c_api_func_import = module_inst->e->c_api_func_imports + func_idx; + if (module_inst->e->common.c_api_func_imports) { + c_api_func_import = + module_inst->e->common.c_api_func_imports + func_idx; func_ptr = c_api_func_import->func_ptr_linked; } else { diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 6698d0e7..3e085248 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -210,8 +210,19 @@ typedef struct CApiFuncImport { void *env_arg; } CApiFuncImport; +/* The common part of WASMModuleInstanceExtra and AOTModuleInstanceExtra */ +typedef struct WASMModuleInstanceExtraCommon { + CApiFuncImport *c_api_func_imports; +#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 + /* Disable bounds checks or not */ + bool disable_bounds_checks; +#endif +} WASMModuleInstanceExtraCommon; + /* Extra info of WASM module instance for interpreter/jit mode */ typedef struct WASMModuleInstanceExtra { + WASMModuleInstanceExtraCommon common; + WASMGlobalInstance *globals; WASMFunctionInstance *functions; @@ -223,7 +234,6 @@ typedef struct WASMModuleInstanceExtra { WASMFunctionInstance *free_function; WASMFunctionInstance *retain_function; - CApiFuncImport *c_api_func_imports; RunningMode running_mode; #if WASM_ENABLE_MULTI_MODULE != 0 @@ -242,10 +252,6 @@ typedef struct WASMModuleInstanceExtra { && WASM_ENABLE_LAZY_JIT != 0) WASMModuleInstance *next; #endif -#if WASM_CONFIGUABLE_BOUNDS_CHECKS != 0 - /* Disable bounds checks or not */ - bool disable_bounds_checks; -#endif } WASMModuleInstanceExtra; struct AOTFuncPerfProfInfo; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 4d1da83a..21425786 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -746,10 +746,10 @@ wasm_cluster_dup_c_api_imports(WASMModuleInstanceCommon *module_inst_dst, #if WASM_ENABLE_INTERP != 0 if (module_inst_src->module_type == Wasm_Module_Bytecode) { - new_c_api_func_imports = - &(((WASMModuleInstance *)module_inst_dst)->e->c_api_func_imports); + new_c_api_func_imports = &(((WASMModuleInstance *)module_inst_dst) + ->e->common.c_api_func_imports); c_api_func_imports = ((const WASMModuleInstance *)module_inst_src) - ->e->c_api_func_imports; + ->e->common.c_api_func_imports; import_func_count = ((WASMModule *)(((const WASMModuleInstance *)module_inst_src) ->module)) @@ -760,10 +760,10 @@ wasm_cluster_dup_c_api_imports(WASMModuleInstanceCommon *module_inst_dst, if (module_inst_src->module_type == Wasm_Module_AoT) { AOTModuleInstanceExtra *e = (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_dst)->e; - new_c_api_func_imports = &(e->c_api_func_imports); + new_c_api_func_imports = &(e->common.c_api_func_imports); e = (AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_src)->e; - c_api_func_imports = e->c_api_func_imports; + c_api_func_imports = e->common.c_api_func_imports; import_func_count = ((AOTModule *)(((AOTModuleInstance *)module_inst_src)->module)) From 7e3a29d5e4564406c91ada436b1e33d53d1232cf Mon Sep 17 00:00:00 2001 From: Cengizhan Pasaoglu Date: Wed, 9 Aug 2023 04:34:21 +0300 Subject: [PATCH 03/34] Strip static and shared libraries of iwasm to reduce the binary size (#2431) Reduce iwasm static and shared library size when CMAKE_STRIP is set and the build type is "Release". It remains as it is when build type was selected Debug. --- CMakeLists.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c879949..fbea2616 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,11 @@ endif () set (WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +# Set the strip command based on the system (GNU or Clang) +if (CMAKE_STRIP) + set (CMAKE_STRIP_FLAGS "--strip-all") +endif () + include (${WAMR_ROOT_DIR}/build-scripts/runtime_lib.cmake) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wformat -Wformat-security -Wshadow -Wno-unused-parameter") @@ -141,6 +146,15 @@ endif () install (TARGETS iwasm_static ARCHIVE DESTINATION lib) +# If it's a Release build, strip the static library +if (CMAKE_STRIP AND CMAKE_BUILD_TYPE STREQUAL "Release") + # Strip static library + message (STATUS "Stripping static library after build!") + add_custom_command (TARGET iwasm_static POST_BUILD + COMMAND ${CMAKE_STRIP} ${CMAKE_STRIP_FLAGS} $ + ) +endif () + # SHARED LIBRARY add_library (iwasm_shared SHARED ${WAMR_RUNTIME_LIB_SOURCE}) set_target_properties (iwasm_shared PROPERTIES OUTPUT_NAME iwasm) @@ -162,3 +176,12 @@ install (FILES ${WAMR_ROOT_DIR}/core/iwasm/include/wasm_export.h ${WAMR_ROOT_DIR}/core/iwasm/include/lib_export.h DESTINATION include) + +# If it's a Release build, strip the shared library +if (CMAKE_STRIP AND CMAKE_BUILD_TYPE STREQUAL "Release") + # Strip shared library + message (STATUS "Stripping shared library after build!") + add_custom_command (TARGET iwasm_shared POST_BUILD + COMMAND ${CMAKE_STRIP} ${CMAKE_STRIP_FLAGS} $ + ) +endif () From 8b5bb0009dc2b4d35417ee50878d7cd41ccf5ca7 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 9 Aug 2023 09:43:20 +0800 Subject: [PATCH 04/34] wasm_export.h: Fix struct wasm_val_t (#2435) Struct wasm_val_t should be same in wasm_export.h and wasm_c_api.h. And fix some invalid calls to aot function in LLVM JIT mode. --- core/iwasm/include/wasm_export.h | 2 + core/iwasm/interpreter/wasm_runtime.c | 71 +++++---------------------- 2 files changed, 13 insertions(+), 60 deletions(-) diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 4d02e027..46bd3782 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -186,6 +186,7 @@ enum wasm_valkind_enum { #ifndef WASM_VAL_T_DEFINED #define WASM_VAL_T_DEFINED +struct wasm_ref_t; typedef struct wasm_val_t { wasm_valkind_t kind; @@ -197,6 +198,7 @@ typedef struct wasm_val_t { double f64; /* represent a foreign object, aka externref in .wat */ uintptr_t foreign; + struct wasm_ref_t *ref; } of; } wasm_val_t; #endif diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 0fdef512..b94be328 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -3093,11 +3093,7 @@ llvm_jit_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 elem_idx, { bool ret; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == exec_env->module_inst->module_type) { - return aot_call_indirect(exec_env, tbl_idx, elem_idx, argc, argv); - } -#endif + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); ret = call_indirect(exec_env, tbl_idx, elem_idx, argc, argv, false, 0); #ifdef OS_ENABLE_HW_BOUND_CHECK @@ -3124,11 +3120,7 @@ llvm_jit_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc, char buf[96]; bool ret = false; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == exec_env->module_inst->module_type) { - return aot_invoke_native(exec_env, func_idx, argc, argv); - } -#endif + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); module_inst = (WASMModuleInstance *)wasm_runtime_get_module_inst(exec_env); module = module_inst->module; @@ -3198,11 +3190,7 @@ llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, uint8 *maddr; uint64 seg_len = 0; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - return aot_memory_init(module_inst, seg_index, offset, len, dst); - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); memory_inst = wasm_get_default_memory(module_inst); module = module_inst->module; @@ -3228,11 +3216,7 @@ llvm_jit_memory_init(WASMModuleInstance *module_inst, uint32 seg_index, bool llvm_jit_data_drop(WASMModuleInstance *module_inst, uint32 seg_index) { -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - return aot_data_drop(module_inst, seg_index); - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); module_inst->module->data_segments[seg_index]->data_length = 0; /* Currently we can't free the dropped data segment @@ -3247,11 +3231,7 @@ llvm_jit_drop_table_seg(WASMModuleInstance *module_inst, uint32 tbl_seg_idx) { WASMTableSeg *tbl_segs; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - return aot_drop_table_seg(module_inst, tbl_seg_idx); - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); tbl_segs = module_inst->module->table_segments; tbl_segs[tbl_seg_idx].is_dropped = true; @@ -3265,12 +3245,7 @@ llvm_jit_table_init(WASMModuleInstance *module_inst, uint32 tbl_idx, WASMTableInstance *tbl_inst; WASMTableSeg *tbl_seg; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - return aot_table_init(module_inst, tbl_idx, tbl_seg_idx, length, - src_offset, dst_offset); - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); tbl_seg = module_inst->module->table_segments + tbl_seg_idx; @@ -3313,13 +3288,7 @@ llvm_jit_table_copy(WASMModuleInstance *module_inst, uint32 src_tbl_idx, WASMTableInstance *src_tbl_inst; WASMTableInstance *dst_tbl_inst; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - aot_table_copy(module_inst, src_tbl_idx, dst_tbl_idx, length, - src_offset, dst_offset); - return; - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); src_tbl_inst = wasm_get_table_inst(module_inst, src_tbl_idx); dst_tbl_inst = wasm_get_table_inst(module_inst, dst_tbl_idx); @@ -3350,12 +3319,7 @@ llvm_jit_table_fill(WASMModuleInstance *module_inst, uint32 tbl_idx, { WASMTableInstance *tbl_inst; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - aot_table_fill(module_inst, tbl_idx, length, val, data_offset); - return; - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); bh_assert(tbl_inst); @@ -3377,11 +3341,7 @@ llvm_jit_table_grow(WASMModuleInstance *module_inst, uint32 tbl_idx, WASMTableInstance *tbl_inst; uint32 i, orig_size, total_size; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == module_inst->module_type) { - return aot_table_grow(module_inst, tbl_idx, inc_size, init_val); - } -#endif + bh_assert(module_inst->module_type == Wasm_Module_Bytecode); tbl_inst = wasm_get_table_inst(module_inst, tbl_idx); if (!tbl_inst) { @@ -3421,11 +3381,7 @@ llvm_jit_alloc_frame(WASMExecEnv *exec_env, uint32 func_index) WASMInterpFrame *frame; uint32 size; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == exec_env->module_inst->module_type) { - return aot_alloc_frame(exec_env, func_index); - } -#endif + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); module_inst = (WASMModuleInstance *)exec_env->module_inst; size = wasm_interp_interp_frame_size(0); @@ -3454,12 +3410,7 @@ llvm_jit_free_frame(WASMExecEnv *exec_env) WASMInterpFrame *frame; WASMInterpFrame *prev_frame; -#if WASM_ENABLE_JIT != 0 - if (Wasm_Module_AoT == exec_env->module_inst->module_type) { - aot_free_frame(exec_env); - return; - } -#endif + bh_assert(exec_env->module_inst->module_type == Wasm_Module_Bytecode); frame = wasm_exec_env_get_cur_frame(exec_env); prev_frame = frame->prev_frame; From f34e7431fd3bd077f5f18e64c858a79910438b31 Mon Sep 17 00:00:00 2001 From: Florian Castellane <583664+LIII-XXII@users.noreply.github.com> Date: Wed, 9 Aug 2023 15:31:02 +0900 Subject: [PATCH 05/34] Fix typos in wamrc print_help() (#2442) --- wamr-compiler/main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 1d3cedf7..8d2c9357 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -126,7 +126,7 @@ print_help() printf(" Use --cpu-features=+help to list all the features supported\n"); printf(" --opt-level=n Set the optimization level (0 to 3, default is 3)\n"); printf(" --size-level=n Set the code size level (0 to 3, default is 3)\n"); - printf(" -sgx Generate code for SGX platform (Intel Software Guard Extention)\n"); + printf(" -sgx Generate code for SGX platform (Intel Software Guard Extensions)\n"); printf(" --bounds-checks=1/0 Enable or disable the bounds checks for memory access:\n"); printf(" by default it is disabled in all 64-bit platforms except SGX and\n"); printf(" in these platforms runtime does bounds checks with hardware trap,\n"); @@ -159,8 +159,8 @@ print_help() printf(" --enable-dump-call-stack Enable stack trace feature\n"); printf(" --enable-perf-profiling Enable function performance profiling\n"); printf(" --enable-memory-profiling Enable memory usage profiling\n"); - printf(" --xip A shorthand of --enalbe-indirect-mode --disable-llvm-intrinsics\n"); - printf(" --enable-indirect-mode Enalbe call function through symbol table but not direct call\n"); + printf(" --xip A shorthand of --enable-indirect-mode --disable-llvm-intrinsics\n"); + printf(" --enable-indirect-mode Enable call function through symbol table but not direct call\n"); printf(" --disable-llvm-intrinsics Disable the LLVM built-in intrinsics\n"); printf(" --enable-builtin-intrinsics=\n"); printf(" Enable the specified built-in intrinsics, it will override the default\n"); From 490fa2ddaccd73834a69e60ce248620e1e0b0c40 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 9 Aug 2023 19:43:08 +0800 Subject: [PATCH 06/34] Auto-check wrgsbase in cmake script (#2437) Auto-check whether `WRGSBASE` instruction is supported in linux x86-64 in the cmake script. And if not, disable writing x86 GS register. --- build-scripts/config_common.cmake | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 6b4247e0..73ed83c9 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -395,6 +395,30 @@ endif () if (WAMR_DISABLE_WRITE_GS_BASE EQUAL 1) add_definitions (-DWASM_DISABLE_WRITE_GS_BASE=1) message (" Write linear memory base addr to x86 GS register disabled") +elseif (WAMR_BUILD_TARGET STREQUAL "X86_64" + AND WAMR_BUILD_PLATFORM STREQUAL "linux") + set (TEST_WRGSBASE_SOURCE "${CMAKE_BINARY_DIR}/test_wrgsbase.c") + file (WRITE "${TEST_WRGSBASE_SOURCE}" " + #include + #include + int main() { + uint64_t value; + asm volatile (\"wrgsbase %0\" : : \"r\"(value)); + printf(\"WRGSBASE instruction is available.\\n\"); + return 0; + }") + # Try to compile and run the test program + try_run (TEST_WRGSBASE_RESULT + TEST_WRGSBASE_COMPILED + ${CMAKE_BINARY_DIR}/test_wrgsbase + SOURCES ${TEST_WRGSBASE_SOURCE} + CMAKE_FLAGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + ) + #message("${TEST_WRGSBASE_COMPILED}, ${TEST_WRGSBASE_RESULT}") + if (NOT TEST_WRGSBASE_RESULT EQUAL 0) + add_definitions (-DWASM_DISABLE_WRITE_GS_BASE=1) + message (" Write linear memory base addr to x86 GS register disabled") + endif () endif () if (WAMR_CONFIGUABLE_BOUNDS_CHECKS EQUAL 1) add_definitions (-DWASM_CONFIGURABLE_BOUNDS_CHECKS=1) From a550f4d9f7a346bc217efe3e89e4f55a44466315 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 10 Aug 2023 10:26:52 +0900 Subject: [PATCH 07/34] iwasm: call native lib init/deinit if exists (#2439) --- product-mini/platforms/posix/main.c | 20 ++++++++++++++++++++ samples/native-lib/README.md | 4 +++- samples/native-lib/test_hello2.c | 13 +++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 45214243..0cf41909 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -284,6 +284,8 @@ validate_env_str(char *env) #if BH_HAS_DLFCN typedef uint32 (*get_native_lib_func)(char **p_module_name, NativeSymbol **p_native_symbols); +typedef int (*init_native_lib_func)(void); +typedef void (*deinit_native_lib_func)(void); static uint32 load_and_register_native_libs(const char **native_lib_list, @@ -304,6 +306,18 @@ load_and_register_native_libs(const char **native_lib_list, continue; } + init_native_lib_func init_native_lib = dlsym(handle, "init_native_lib"); + if (init_native_lib) { + int ret = init_native_lib(); + if (ret != 0) { + LOG_WARNING("warning: `init_native_lib` function from native " + "lib %s failed with %d", + native_lib_list[i], ret); + dlclose(handle); + continue; + } + } + /* lookup get_native_lib func */ get_native_lib_func get_native_lib = dlsym(handle, "get_native_lib"); if (!get_native_lib) { @@ -368,6 +382,12 @@ unregister_and_unload_native_libs(uint32 native_lib_count, continue; } + deinit_native_lib_func deinit_native_lib = + dlsym(handle, "deinit_native_lib"); + if (deinit_native_lib) { + deinit_native_lib(); + } + dlclose(handle); } } diff --git a/samples/native-lib/README.md b/samples/native-lib/README.md index 2bf65814..80500ade 100644 --- a/samples/native-lib/README.md +++ b/samples/native-lib/README.md @@ -62,6 +62,7 @@ cd build The output is: ```bash +init_native_lib in test_hello2.c called Hello World! 10 + 20 = 30 sqrt(10, 20) = 500 @@ -72,5 +73,6 @@ Message from test_hello: Hello, main. This is test_hello_wrapper! test_hello2("main", 0x0, 0) = 85 malloc(86) = 0x24e8 test_hello2("main", 0x24e8, 86) = 85 -Message from test_hello2: Hello, main. This is test_hello2_wrapper! Your wasm_module_inst_t is 0x7fd443704990. +Message from test_hello2: Hello, main. This is test_hello2_wrapper! Your wasm_module_inst_t is 0x7fe0e6804280. +deinit_native_lib in test_hello2.c called ``` diff --git a/samples/native-lib/test_hello2.c b/samples/native-lib/test_hello2.c index 2c8f69ed..5dae79ca 100644 --- a/samples/native-lib/test_hello2.c +++ b/samples/native-lib/test_hello2.c @@ -57,3 +57,16 @@ get_native_lib(char **p_module_name, NativeSymbol **p_native_symbols) *p_native_symbols = native_symbols; return sizeof(native_symbols) / sizeof(NativeSymbol); } + +int +init_native_lib() +{ + printf("%s in test_hello2.c called\n", __func__); + return 0; +} + +void +deinit_native_lib() +{ + printf("%s in test_hello2.c called\n", __func__); +} From 0b0af1b3df79e6c8b000c2dbcec36940dd195990 Mon Sep 17 00:00:00 2001 From: tonibofarull Date: Fri, 11 Aug 2023 01:55:40 +0200 Subject: [PATCH 08/34] wasi-nn: Support uint8 quantized networks (#2433) Support (non-full) uint8 quantized networks. Inputs and outputs are still required to be `float`. The (de)quantization is done internally by wasi-nn. Example generated from `quantized_model.py`: ![Screenshot from 2023-08-07 17-57-05](https://github.com/bytecodealliance/wasm-micro-runtime/assets/80318361/91f12ff6-870c-427a-b1dc-e307f7d1f5ee) Visualization with [netron](https://netron.app/). --- core/iwasm/libraries/wasi-nn/.gitignore | 2 + .../wasi-nn/src/wasi_nn_tensorflowlite.cpp | 74 +++++++++++++++---- .../libraries/wasi-nn/test/Dockerfile.cpu | 1 - .../wasi-nn/test/Dockerfile.nvidia-gpu | 1 - core/iwasm/libraries/wasi-nn/test/build.sh | 22 +++++- .../wasi-nn/test/models/quantized.py | 30 ++++++++ .../wasi-nn/test/test_tensorflow_quantized.c | 63 ++++++++++++++++ 7 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 core/iwasm/libraries/wasi-nn/.gitignore create mode 100644 core/iwasm/libraries/wasi-nn/test/models/quantized.py create mode 100644 core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c diff --git a/core/iwasm/libraries/wasi-nn/.gitignore b/core/iwasm/libraries/wasi-nn/.gitignore new file mode 100644 index 00000000..81c0ff6a --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/.gitignore @@ -0,0 +1,2 @@ +**/*.wasm +**/*.tflite diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp index d9fb2668..b6fda1eb 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp @@ -285,14 +285,37 @@ tensorflowlite_set_input(void *tflite_ctx, graph_execution_context ctx, return invalid_argument; } - auto *input = - tfl_ctx->interpreters[ctx].interpreter->typed_input_tensor( - index); - if (input == NULL) - return missing_memory; + if (tensor->quantization.type == kTfLiteNoQuantization) { + NN_DBG_PRINTF("No quantization information. Using float as default"); + float *it = + tfl_ctx->interpreters[ctx].interpreter->typed_input_tensor( + index); + + int size = model_tensor_size * sizeof(float); + bh_memcpy_s(it, size, input_tensor->data, size); + } + else { // TODO: Assumming uint8 quantized networks. + TfLiteAffineQuantization *quant_info = + (TfLiteAffineQuantization *)tensor->quantization.params; + if (quant_info->scale->size != 1 || quant_info->zero_point->size != 1) { + NN_ERR_PRINTF("Quantization per channel is not supported"); + return runtime_error; + } + uint8_t *it = + tfl_ctx->interpreters[ctx].interpreter->typed_input_tensor( + index); + + float scale = quant_info->scale->data[0]; + float zero_point = (float)quant_info->zero_point->data[0]; + NN_DBG_PRINTF("input tensor: (scale, offset) = (%f, %f)", scale, + zero_point); + + float *input_tensor_f = (float *)input_tensor->data; + for (uint32_t i = 0; i < model_tensor_size; ++i) { + it[i] = (uint8_t)(input_tensor_f[i] / scale + zero_point); + } + } - bh_memcpy_s(input, model_tensor_size * sizeof(float), input_tensor->data, - model_tensor_size * sizeof(float)); return success; } @@ -325,6 +348,7 @@ tensorflowlite_get_output(void *tflite_ctx, graph_execution_context ctx, NN_DBG_PRINTF("Number of tensors (%d)", num_output_tensors); if (index + 1 > num_output_tensors) { + NN_ERR_PRINTF("Index %d is invalid.", index); return runtime_error; } @@ -343,15 +367,37 @@ tensorflowlite_get_output(void *tflite_ctx, graph_execution_context ctx, return missing_memory; } - float *tensor_f = - tfl_ctx->interpreters[ctx].interpreter->typed_output_tensor( - index); - for (uint32_t i = 0; i < model_tensor_size; ++i) - NN_DBG_PRINTF("output: %f", tensor_f[i]); + if (tensor->quantization.type == kTfLiteNoQuantization) { + NN_DBG_PRINTF("No quantization information"); + float *ot = + tfl_ctx->interpreters[ctx].interpreter->typed_output_tensor( + index); + + int size = model_tensor_size * sizeof(float); + bh_memcpy_s(output_tensor, size, ot, size); + } + else { // TODO: Assumming uint8 quantized networks. + TfLiteAffineQuantization *quant_info = + (TfLiteAffineQuantization *)tensor->quantization.params; + if (quant_info->scale->size != 1 || quant_info->zero_point->size != 1) { + NN_ERR_PRINTF("Quantization per channel is not supported"); + return runtime_error; + } + uint8_t *ot = tfl_ctx->interpreters[ctx] + .interpreter->typed_output_tensor(index); + + float scale = quant_info->scale->data[0]; + float zero_point = (float)quant_info->zero_point->data[0]; + NN_DBG_PRINTF("output tensor: (scale, offset) = (%f, %f)", scale, + zero_point); + + float *output_tensor_f = (float *)output_tensor; + for (uint32_t i = 0; i < model_tensor_size; ++i) { + output_tensor_f[i] = (ot[i] - zero_point) * scale; + } + } *output_tensor_size = model_tensor_size; - bh_memcpy_s(output_tensor, model_tensor_size * sizeof(float), tensor_f, - model_tensor_size * sizeof(float)); return success; } diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu index f98f121e..6fe56567 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.cpu @@ -30,7 +30,6 @@ RUN make -j "$(grep -c ^processor /proc/cpuinfo)" FROM ubuntu:22.04 -COPY --from=base /home/wamr/product-mini/platforms/linux/build/libvmlib.so /libvmlib.so COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /iwasm ENTRYPOINT [ "/iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu index be80ad45..d78021ea 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu @@ -44,7 +44,6 @@ RUN mkdir -p /etc/OpenCL/vendors && \ ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility -COPY --from=base /home/wamr/product-mini/platforms/linux/build/libvmlib.so /libvmlib.so COPY --from=base /home/wamr/product-mini/platforms/linux/build/iwasm /iwasm ENTRYPOINT [ "/iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/build.sh b/core/iwasm/libraries/wasi-nn/test/build.sh index f018f600..da6cfa5f 100755 --- a/core/iwasm/libraries/wasi-nn/test/build.sh +++ b/core/iwasm/libraries/wasi-nn/test/build.sh @@ -1,6 +1,10 @@ +#!/bin/sh + # Copyright (C) 2019 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +CURR_PATH=$(cd $(dirname $0) && pwd -P) + # WASM application that uses WASI-NN /opt/wasi-sdk/bin/clang \ @@ -13,9 +17,25 @@ # TFLite models to use in the tests -cd models +cd ${CURR_PATH}/models python3 average.py python3 max.py python3 mult_dimension.py python3 mult_outputs.py python3 sum.py + +# Specific tests for TPU + +cd ${CURR_PATH} +/opt/wasi-sdk/bin/clang \ + -Wl,--allow-undefined \ + -Wl,--strip-all,--no-entry \ + --sysroot=/opt/wasi-sdk/share/wasi-sysroot \ + -I../include -I../src/utils \ + -o test_tensorflow_quantized.wasm \ + test_tensorflow_quantized.c utils.c + +cd ${CURR_PATH}/models +python3 quantized.py + +cd ${CURR_PATH} diff --git a/core/iwasm/libraries/wasi-nn/test/models/quantized.py b/core/iwasm/libraries/wasi-nn/test/models/quantized.py new file mode 100644 index 00000000..b195b319 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/test/models/quantized.py @@ -0,0 +1,30 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import tensorflow as tf +import numpy as np +import pathlib + +model = tf.keras.Sequential([ + tf.keras.layers.InputLayer(input_shape=[5, 5, 1]), + tf.keras.layers.AveragePooling2D( + pool_size=(5, 5), strides=None, padding="valid", data_format=None) + +]) + +def representative_dataset(): + for _ in range(1000): + data = np.random.randint(0, 25, (1, 5, 5, 1)) + yield [data.astype(np.float32)] + +converter = tf.lite.TFLiteConverter.from_keras_model(model) +converter.optimizations = [tf.lite.Optimize.DEFAULT] +converter.representative_dataset = representative_dataset +converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] +converter.inference_input_type = tf.uint8 # or tf.int8 +converter.inference_output_type = tf.uint8 # or tf.int8 +tflite_model = converter.convert() + +tflite_models_dir = pathlib.Path("./") +tflite_model_file = tflite_models_dir / "quantized_model.tflite" +tflite_model_file.write_bytes(tflite_model) diff --git a/core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c b/core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c new file mode 100644 index 00000000..3ed7c751 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/test/test_tensorflow_quantized.c @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +#include "utils.h" +#include "logger.h" + +#undef EPSILON +#define EPSILON 1e-2 + +void +test_average_quantized(execution_target target) +{ + int dims[] = { 1, 5, 5, 1 }; + input_info input = create_input(dims); + + uint32_t output_size = 0; + float *output = + run_inference(target, input.input_tensor, input.dim, &output_size, + "./models/quantized_model.tflite", 1); + + NN_INFO_PRINTF("Output size: %d", output_size); + NN_INFO_PRINTF("Result: average is %f", output[0]); + // NOTE: 11.95 instead of 12 because of errors due quantization + assert(fabs(output[0] - 11.95) < EPSILON); + + free(input.dim); + free(input.input_tensor); + free(output); +} + +int +main() +{ + char *env = getenv("TARGET"); + if (env == NULL) { + NN_INFO_PRINTF("Usage:\n--env=\"TARGET=[cpu|gpu|tpu]\""); + return 1; + } + execution_target target; + if (strcmp(env, "cpu") == 0) + target = cpu; + else if (strcmp(env, "gpu") == 0) + target = gpu; + else if (strcmp(env, "tpu") == 0) + target = tpu; + else { + NN_ERR_PRINTF("Wrong target!"); + return 1; + } + NN_INFO_PRINTF("################### Testing quantized model..."); + test_average_quantized(target); + + NN_INFO_PRINTF("Tests: passed!"); + return 0; +} From 5780effc07126d30cefa6c30e6197ae16413194b Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 14 Aug 2023 15:05:59 +0900 Subject: [PATCH 09/34] iwasm: Fix native lib cleanup after error occurs (#2443) - If something failed after calling init_native_lib, call deinit_native_lib to clean up. - Restructure the relevant code for better maintainability. --- product-mini/platforms/posix/main.c | 186 +++++++++++++++------------- 1 file changed, 99 insertions(+), 87 deletions(-) diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index 0cf41909..b79af9b8 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -282,113 +282,124 @@ validate_env_str(char *env) #endif #if BH_HAS_DLFCN -typedef uint32 (*get_native_lib_func)(char **p_module_name, - NativeSymbol **p_native_symbols); -typedef int (*init_native_lib_func)(void); -typedef void (*deinit_native_lib_func)(void); +struct native_lib { + void *handle; + + uint32 (*get_native_lib)(char **p_module_name, + NativeSymbol **p_native_symbols); + int (*init_native_lib)(void); + void (*deinit_native_lib)(void); + + char *module_name; + NativeSymbol *native_symbols; + uint32 n_native_symbols; +}; + +struct native_lib * +load_native_lib(const char *name) +{ + struct native_lib *lib = wasm_runtime_malloc(sizeof(*lib)); + if (lib == NULL) { + LOG_WARNING("warning: failed to load native library %s because of " + "allocation failure", + name); + goto fail; + } + memset(lib, 0, sizeof(*lib)); + + /* open the native library */ + if (!(lib->handle = dlopen(name, RTLD_NOW | RTLD_GLOBAL)) + && !(lib->handle = dlopen(name, RTLD_LAZY))) { + LOG_WARNING("warning: failed to load native library %s", name); + goto fail; + } + + lib->init_native_lib = dlsym(lib->handle, "init_native_lib"); + lib->get_native_lib = dlsym(lib->handle, "get_native_lib"); + lib->deinit_native_lib = dlsym(lib->handle, "deinit_native_lib"); + + if (!lib->get_native_lib) { + LOG_WARNING("warning: failed to lookup `get_native_lib` function " + "from native lib %s", + name); + goto fail; + } + + if (lib->init_native_lib) { + int ret = lib->init_native_lib(); + if (ret != 0) { + LOG_WARNING("warning: `init_native_lib` function from native " + "lib %s failed with %d", + name); + goto fail; + } + } + + lib->n_native_symbols = + lib->get_native_lib(&lib->module_name, &lib->native_symbols); + + /* register native symbols */ + if (!(lib->n_native_symbols > 0 && lib->module_name && lib->native_symbols + && wasm_runtime_register_natives( + lib->module_name, lib->native_symbols, lib->n_native_symbols))) { + LOG_WARNING("warning: failed to register native lib %s", name); + if (lib->deinit_native_lib) { + lib->deinit_native_lib(); + } + goto fail; + } + return lib; +fail: + if (lib != NULL) { + if (lib->handle != NULL) { + dlclose(lib->handle); + } + wasm_runtime_free(lib); + } + return NULL; +} static uint32 load_and_register_native_libs(const char **native_lib_list, uint32 native_lib_count, - void **native_handle_list) + struct native_lib **native_lib_loaded_list) { - uint32 i, native_handle_count = 0, n_native_symbols; - NativeSymbol *native_symbols; - char *module_name; - void *handle; + uint32 i, native_lib_loaded_count = 0; for (i = 0; i < native_lib_count; i++) { - /* open the native library */ - if (!(handle = dlopen(native_lib_list[i], RTLD_NOW | RTLD_GLOBAL)) - && !(handle = dlopen(native_lib_list[i], RTLD_LAZY))) { - LOG_WARNING("warning: failed to load native library %s", - native_lib_list[i]); + struct native_lib *lib = load_native_lib(native_lib_list[i]); + if (lib == NULL) { continue; } - - init_native_lib_func init_native_lib = dlsym(handle, "init_native_lib"); - if (init_native_lib) { - int ret = init_native_lib(); - if (ret != 0) { - LOG_WARNING("warning: `init_native_lib` function from native " - "lib %s failed with %d", - native_lib_list[i], ret); - dlclose(handle); - continue; - } - } - - /* lookup get_native_lib func */ - get_native_lib_func get_native_lib = dlsym(handle, "get_native_lib"); - if (!get_native_lib) { - LOG_WARNING("warning: failed to lookup `get_native_lib` function " - "from native lib %s", - native_lib_list[i]); - dlclose(handle); - continue; - } - - n_native_symbols = get_native_lib(&module_name, &native_symbols); - - /* register native symbols */ - if (!(n_native_symbols > 0 && module_name && native_symbols - && wasm_runtime_register_natives(module_name, native_symbols, - n_native_symbols))) { - LOG_WARNING("warning: failed to register native lib %s", - native_lib_list[i]); - dlclose(handle); - continue; - } - - native_handle_list[native_handle_count++] = handle; + native_lib_loaded_list[native_lib_loaded_count++] = lib; } - return native_handle_count; + return native_lib_loaded_count; } static void unregister_and_unload_native_libs(uint32 native_lib_count, - void **native_handle_list) + struct native_lib **native_lib_loaded_list) { - uint32 i, n_native_symbols; - NativeSymbol *native_symbols; - char *module_name; - void *handle; + uint32 i; for (i = 0; i < native_lib_count; i++) { - handle = native_handle_list[i]; - - /* lookup get_native_lib func */ - get_native_lib_func get_native_lib = dlsym(handle, "get_native_lib"); - if (!get_native_lib) { - LOG_WARNING("warning: failed to lookup `get_native_lib` function " - "from native lib %p", - handle); - continue; - } - - n_native_symbols = get_native_lib(&module_name, &native_symbols); - if (n_native_symbols == 0 || module_name == NULL - || native_symbols == NULL) { - LOG_WARNING("warning: get_native_lib returned different values for " - "native lib %p", - handle); - continue; - } + struct native_lib *lib = native_lib_loaded_list[i]; /* unregister native symbols */ - if (!wasm_runtime_unregister_natives(module_name, native_symbols)) { - LOG_WARNING("warning: failed to unregister native lib %p", handle); + if (!wasm_runtime_unregister_natives(lib->module_name, + lib->native_symbols)) { + LOG_WARNING("warning: failed to unregister native lib %p", + lib->handle); continue; } - deinit_native_lib_func deinit_native_lib = - dlsym(handle, "deinit_native_lib"); - if (deinit_native_lib) { - deinit_native_lib(); + if (lib->deinit_native_lib) { + lib->deinit_native_lib(); } - dlclose(handle); + dlclose(lib->handle); + wasm_runtime_free(lib); } } #endif /* BH_HAS_DLFCN */ @@ -525,8 +536,8 @@ main(int argc, char *argv[]) #if BH_HAS_DLFCN const char *native_lib_list[8] = { NULL }; uint32 native_lib_count = 0; - void *native_handle_list[8] = { NULL }; - uint32 native_handle_count = 0; + struct native_lib *native_lib_loaded_list[8]; + uint32 native_lib_loaded_count = 0; #endif #if WASM_ENABLE_DEBUG_INTERP != 0 char *ip_addr = NULL; @@ -810,8 +821,8 @@ main(int argc, char *argv[]) #endif #if BH_HAS_DLFCN - native_handle_count = load_and_register_native_libs( - native_lib_list, native_lib_count, native_handle_list); + native_lib_loaded_count = load_and_register_native_libs( + native_lib_list, native_lib_count, native_lib_loaded_list); #endif /* load WASM byte buffer from WASM bin file */ @@ -945,7 +956,8 @@ fail2: fail1: #if BH_HAS_DLFCN /* unload the native libraries */ - unregister_and_unload_native_libs(native_handle_count, native_handle_list); + unregister_and_unload_native_libs(native_lib_loaded_count, + native_lib_loaded_list); #endif /* destroy runtime environment */ From 5c6613b2b17184289c86cb0f86c707aa240364f3 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 14 Aug 2023 16:12:59 +0900 Subject: [PATCH 10/34] Correct --heap-size option in messages (#2458) --- core/iwasm/aot/aot_runtime.c | 4 ++-- core/iwasm/interpreter/wasm_runtime.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 94c66e67..2c647d34 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -420,7 +420,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, if (num_bytes_per_page < heap_size) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " - "try using `--heap_size=0` option"); + "try using `--heap-size=0` option"); return NULL; } } @@ -478,7 +478,7 @@ memory_instantiate(AOTModuleInstance *module_inst, AOTModuleInstance *parent, if (init_page_count > DEFAULT_MAX_PAGES) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " - "try using `--heap_size=0` option"); + "try using `--heap-size=0` option"); return NULL; } else if (init_page_count == DEFAULT_MAX_PAGES) { diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index b94be328..63df4cb3 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -202,7 +202,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, if (num_bytes_per_page < heap_size) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " - "try using `--heap_size=0` option"); + "try using `--heap-size=0` option"); return NULL; } } @@ -261,7 +261,7 @@ memory_instantiate(WASMModuleInstance *module_inst, WASMModuleInstance *parent, if (init_page_count > DEFAULT_MAX_PAGES) { set_error_buf(error_buf, error_buf_size, "failed to insert app heap into linear memory, " - "try using `--heap_size=0` option"); + "try using `--heap-size=0` option"); return NULL; } else if (init_page_count == DEFAULT_MAX_PAGES) { From 8d1cf46f02a27f52b24103b5d5b5511dd3990a31 Mon Sep 17 00:00:00 2001 From: tonibofarull Date: Mon, 14 Aug 2023 10:45:30 +0200 Subject: [PATCH 11/34] Implement `wasm_externref_objdel` and `wasm_externref_set_cleanup` (#2455) ## Context Some native libraries may want to explicitly delete an externref object without waiting for the module instance to be deleted. In addition, it may want to add a cleanup function. ## Proposed Changes Implement: * `wasm_externref_objdel` to explicitly delete an externeref'd object. * `wasm_externref_set_cleanup` to set a cleanup function that is called when the externref'd object is deleted. --- core/iwasm/common/wasm_runtime_common.c | 84 +++++++++++++++++++++++-- core/iwasm/include/wasm_export.h | 26 ++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index a39613a3..b1866215 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -4681,6 +4681,8 @@ typedef struct ExternRefMapNode { bool retained; /* Whether it is marked by runtime */ bool marked; + /* cleanup function called when the externref is freed */ + void (*cleanup)(void *); } ExternRefMapNode; static uint32 @@ -4743,6 +4745,81 @@ lookup_extobj_callback(void *key, void *value, void *user_data) } } +static void +delete_externref(void *key, ExternRefMapNode *node) +{ + bh_hash_map_remove(externref_map, key, NULL, NULL); + if (node->cleanup) { + (*node->cleanup)(node->extern_obj); + } + wasm_runtime_free(node); +} + +static void +delete_extobj_callback(void *key, void *value, void *user_data) +{ + ExternRefMapNode *node = (ExternRefMapNode *)value; + LookupExtObj_UserData *lookup_user_data = + (LookupExtObj_UserData *)user_data; + + if (node->extern_obj == lookup_user_data->node.extern_obj + && node->module_inst == lookup_user_data->node.module_inst) { + lookup_user_data->found = true; + delete_externref(key, node); + } +} + +bool +wasm_externref_objdel(WASMModuleInstanceCommon *module_inst, void *extern_obj) +{ + LookupExtObj_UserData lookup_user_data = { 0 }; + bool ok = false; + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, delete_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + ok = true; + } + os_mutex_unlock(&externref_lock); + + return ok; +} + +bool +wasm_externref_set_cleanup(WASMModuleInstanceCommon *module_inst, + void *extern_obj, void (*extern_obj_cleanup)(void *)) +{ + + LookupExtObj_UserData lookup_user_data = { 0 }; + bool ok = false; + + /* in a wrapper, extern_obj could be any value */ + lookup_user_data.node.extern_obj = extern_obj; + lookup_user_data.node.module_inst = module_inst; + lookup_user_data.found = false; + + os_mutex_lock(&externref_lock); + /* Lookup hashmap firstly */ + bh_hash_map_traverse(externref_map, lookup_extobj_callback, + (void *)&lookup_user_data); + if (lookup_user_data.found) { + void *key = (void *)(uintptr_t)lookup_user_data.externref_idx; + ExternRefMapNode *node = bh_hash_map_find(externref_map, key); + node->cleanup = extern_obj_cleanup; + ok = true; + } + os_mutex_unlock(&externref_lock); + + return ok; +} + bool wasm_externref_obj2ref(WASMModuleInstanceCommon *module_inst, void *extern_obj, uint32 *p_externref_idx) @@ -4792,6 +4869,7 @@ wasm_externref_obj2ref(WASMModuleInstanceCommon *module_inst, void *extern_obj, memset(node, 0, sizeof(ExternRefMapNode)); node->extern_obj = extern_obj; node->module_inst = module_inst; + node->cleanup = NULL; externref_idx = externref_global_id; @@ -4842,8 +4920,7 @@ reclaim_extobj_callback(void *key, void *value, void *user_data) if (node->module_inst == module_inst) { if (!node->marked && !node->retained) { - bh_hash_map_remove(externref_map, key, NULL, NULL); - wasm_runtime_free(value); + delete_externref(key, node); } else { node->marked = false; @@ -4958,8 +5035,7 @@ cleanup_extobj_callback(void *key, void *value, void *user_data) (WASMModuleInstanceCommon *)user_data; if (node->module_inst == module_inst) { - bh_hash_map_remove(externref_map, key, NULL, NULL); - wasm_runtime_free(value); + delete_externref(key, node); } } diff --git a/core/iwasm/include/wasm_export.h b/core/iwasm/include/wasm_export.h index 46bd3782..18e15782 100644 --- a/core/iwasm/include/wasm_export.h +++ b/core/iwasm/include/wasm_export.h @@ -1292,6 +1292,32 @@ WASM_RUNTIME_API_EXTERN bool wasm_externref_obj2ref(wasm_module_inst_t module_inst, void *extern_obj, uint32_t *p_externref_idx); +/** + * Delete external object registered by `wasm_externref_obj2ref`. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to be deleted + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_objdel(wasm_module_inst_t module_inst, void *extern_obj); + +/** + * Set cleanup callback to release external object. + * + * @param module_inst the WASM module instance that the extern object + * belongs to + * @param extern_obj the external object to which to set the `extern_obj_cleanup` cleanup callback. + * @param extern_obj_cleanup a callback to release `extern_obj` + * + * @return true if success, false otherwise + */ +WASM_RUNTIME_API_EXTERN bool +wasm_externref_set_cleanup(wasm_module_inst_t module_inst, void *extern_obj, + void (*extern_obj_cleanup)(void *)); + /** * Retrieve the external object from an internal externref index * From e360b7a919247d2ab4d0363e1edf53bd8c073210 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Mon, 14 Aug 2023 18:27:14 +0900 Subject: [PATCH 12/34] wasm_instantiate: Fix a potential integer overflow issue (#2459) Fixes: https://github.com/bytecodealliance/wasm-micro-runtime/issues/2450 --- core/iwasm/interpreter/wasm_runtime.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 63df4cb3..2b88c398 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1839,7 +1839,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, for (i = 0; i < module->data_seg_count; i++) { WASMMemoryInstance *memory = NULL; uint8 *memory_data = NULL; - uint32 memory_size = 0; + uint64 memory_size = 0; WASMDataSeg *data_seg = module->data_segments[i]; #if WASM_ENABLE_BULK_MEMORY != 0 @@ -1852,7 +1852,8 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, bh_assert(memory); memory_data = memory->memory_data; - memory_size = memory->num_bytes_per_page * memory->cur_page_count; + memory_size = + (uint64)memory->num_bytes_per_page * memory->cur_page_count; bh_assert(memory_data || memory_size == 0); bh_assert(data_seg->base_offset.init_expr_type @@ -1898,7 +1899,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, /* check offset + length(could be zero) */ length = data_seg->data_length; - if (base_offset + length > memory_size) { + if ((uint64)base_offset + length > memory_size) { LOG_DEBUG("base_offset(%d) + length(%d) > memory_size(%d)", base_offset, length, memory_size); #if WASM_ENABLE_REF_TYPES != 0 From 923e8d64713e3349f192d75ff61c5e0591f32845 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Mon, 14 Aug 2023 19:04:49 +0800 Subject: [PATCH 13/34] Fix windows link error and clear windows warnings (#2463) - Fix windows wamrc link error: aot_generate_tempfile_name undefined. - Clear windows compile warnings. - And rename folder `samples/bh_atomic` and `samples/mem_allocator` to `samples/bh-atomic` and `samples/mem-allocator`. --- core/iwasm/compilation/aot_compiler.c | 27 +++++++++++++++++++ core/iwasm/compilation/aot_emit_aot_file.c | 18 ++++++------- core/iwasm/compilation/aot_llvm.c | 19 ++++++------- .../{bh_atomic => bh-atomic}/CMakeLists.txt | 0 samples/{bh_atomic => bh-atomic}/main.c | 0 .../CMakeLists.txt | 0 .../{mem_allocator => mem-allocator}/main.c | 0 7 files changed, 46 insertions(+), 18 deletions(-) rename samples/{bh_atomic => bh-atomic}/CMakeLists.txt (100%) rename samples/{bh_atomic => bh-atomic}/main.c (100%) rename samples/{mem_allocator => mem-allocator}/CMakeLists.txt (100%) rename samples/{mem_allocator => mem-allocator}/main.c (100%) diff --git a/core/iwasm/compilation/aot_compiler.c b/core/iwasm/compilation/aot_compiler.c index e99e2ae7..464ca61f 100644 --- a/core/iwasm/compilation/aot_compiler.c +++ b/core/iwasm/compilation/aot_compiler.c @@ -2735,6 +2735,33 @@ aot_generate_tempfile_name(const char *prefix, const char *extension, snprintf(buffer + name_len, len - name_len, ".%s", extension); return buffer; } +#else + +errno_t +_mktemp_s(char *nameTemplate, size_t sizeInChars); + +char * +aot_generate_tempfile_name(const char *prefix, const char *extension, + char *buffer, uint32 len) +{ + int name_len; + + name_len = snprintf(buffer, len, "%s-XXXXXX", prefix); + + if (_mktemp_s(buffer, name_len + 1) != 0) { + return NULL; + } + + /* Check if buffer length is enough */ + /* name_len + '.' + extension + '\0' */ + if (name_len + 1 + strlen(extension) + 1 > len) { + aot_set_last_error("temp file name too long."); + return NULL; + } + + snprintf(buffer + name_len, len - name_len, ".%s", extension); + return buffer; +} #endif /* end of !(defined(_WIN32) || defined(_WIN32_)) */ bool diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index bebae550..f1a5c479 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -2403,7 +2403,7 @@ aot_resolve_object_data_sections(AOTObjectData *obj_data) && !strcmp(name, "__llvm_prf_cnts")) { snprintf(buf, sizeof(buf), "%s%u", name, llvm_prf_cnts_idx++); - size = strlen(buf) + 1; + size = (uint32)(strlen(buf) + 1); if (!(data_section->name = wasm_runtime_malloc(size))) { aot_set_last_error( "allocate memory for data section name failed."); @@ -2416,7 +2416,7 @@ aot_resolve_object_data_sections(AOTObjectData *obj_data) && !strcmp(name, "__llvm_prf_data")) { snprintf(buf, sizeof(buf), "%s%u", name, llvm_prf_data_idx++); - size = strlen(buf) + 1; + size = (uint32)(strlen(buf) + 1); if (!(data_section->name = wasm_runtime_malloc(size))) { aot_set_last_error( "allocate memory for data section name failed."); @@ -2520,15 +2520,15 @@ read_stack_usage_file(const AOTCompContext *comp_ctx, const char *filename, } if (prefix == aot_func_prefix) { if (sz < precheck_stack_size_min) { - precheck_stack_size_min = sz; + precheck_stack_size_min = (uint32)sz; } if (sz > precheck_stack_size_max) { - precheck_stack_size_max = sz; + precheck_stack_size_max = (uint32)sz; } precheck_found++; continue; } - sizes[func_idx] = sz; + sizes[func_idx] = (uint32)sz; found++; } fclose(fp); @@ -2651,7 +2651,7 @@ aot_resolve_stack_sizes(AOTCompContext *comp_ctx, AOTObjectData *obj_data) * aot_emit_object_data_section_info will emit this copy. */ obj_data->stack_sizes_section_name = sec_name; - obj_data->stack_sizes_offset = addr; + obj_data->stack_sizes_offset = (uint32)addr; obj_data->stack_sizes = wasm_runtime_malloc( obj_data->func_count * sizeof(*obj_data->stack_sizes)); if (obj_data->stack_sizes == NULL) { @@ -2975,7 +2975,7 @@ aot_resolve_object_relocation_group(AOTObjectData *obj_data, || !strcmp(group->section_name, ".rel.text")) { snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name, prof_section_idx); - size = strlen(buf) + 1; + size = (uint32)(strlen(buf) + 1); if (!(relocation->symbol_name = wasm_runtime_malloc(size))) { aot_set_last_error( "allocate memory for relocation symbol name failed."); @@ -2990,7 +2990,7 @@ aot_resolve_object_relocation_group(AOTObjectData *obj_data, 19)) { snprintf(buf, sizeof(buf), "%s%u", relocation->symbol_name, prof_section_idx); - size = strlen(buf) + 1; + size = (uint32)(strlen(buf) + 1); if (!(relocation->symbol_name = wasm_runtime_malloc(size))) { aot_set_last_error( "allocate memory for relocation symbol name failed."); @@ -3156,7 +3156,7 @@ aot_resolve_object_relocation_groups(AOTObjectData *obj_data) || !strcmp(name, ".rel__llvm_prf_data"))) { char buf[32]; snprintf(buf, sizeof(buf), "%s%u", name, llvm_prf_data_idx); - size = strlen(buf) + 1; + size = (uint32)(strlen(buf) + 1); if (!(relocation_group->section_name = wasm_runtime_malloc(size))) { aot_set_last_error( diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index e066f7a6..b47c198c 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -237,9 +237,10 @@ get_inst_extra_offset(AOTCompContext *comp_ctx) const AOTCompData *comp_data = comp_ctx->comp_data; uint32 table_count = comp_data->import_table_count + comp_data->table_count; uint64 offset = get_tbl_inst_offset(comp_ctx, NULL, table_count); - bh_assert(offset <= UINT_MAX); - offset = align_uint(offset, 8); - return offset; + uint32 offset_32 = (uint32)offset; + bh_assert(offset <= UINT32_MAX); + offset_32 = align_uint((uint32)offset_32, 8); + return offset_32; } /* @@ -309,8 +310,8 @@ aot_add_precheck_function(AOTCompContext *comp_ctx, LLVMModuleRef module, goto fail; } - unsigned int param_count = LLVMCountParams(precheck_func); - uint64 sz = param_count * sizeof(LLVMValueRef); + uint32 param_count = LLVMCountParams(precheck_func); + uint32 sz = param_count * (uint32)sizeof(LLVMValueRef); params = wasm_runtime_malloc(sz); if (params == NULL) { goto fail; @@ -626,8 +627,8 @@ aot_add_llvm_func(AOTCompContext *comp_ctx, LLVMModuleRef module, if (comp_ctx->is_indirect_mode) { /* avoid LUT relocations ("switch-table") */ LLVMAttributeRef attr_no_jump_tables = LLVMCreateStringAttribute( - comp_ctx->context, "no-jump-tables", strlen("no-jump-tables"), - "true", strlen("true")); + comp_ctx->context, "no-jump-tables", + (uint32)strlen("no-jump-tables"), "true", (uint32)strlen("true")); LLVMAddAttributeAtIndex(func, LLVMAttributeFunctionIndex, attr_no_jump_tables); } @@ -2081,7 +2082,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, return; } /* ensure NUL termination */ - bh_memcpy_s(buf, sizeof(buf), name, namelen); + bh_memcpy_s(buf, (uint32)sizeof(buf), name, (uint32)namelen); buf[namelen] = 0; ret = sscanf(buf, AOT_FUNC_INTERNAL_PREFIX "%" SCNu32, &func_idx); @@ -2102,7 +2103,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, /* Note: -1 == AOT_NEG_ONE from aot_create_stack_sizes */ bh_assert(comp_ctx->jit_stack_sizes[func_idx] == (uint32)-1); - comp_ctx->jit_stack_sizes[func_idx] = stack_size + call_size; + comp_ctx->jit_stack_sizes[func_idx] = (uint32)stack_size + call_size; } static bool diff --git a/samples/bh_atomic/CMakeLists.txt b/samples/bh-atomic/CMakeLists.txt similarity index 100% rename from samples/bh_atomic/CMakeLists.txt rename to samples/bh-atomic/CMakeLists.txt diff --git a/samples/bh_atomic/main.c b/samples/bh-atomic/main.c similarity index 100% rename from samples/bh_atomic/main.c rename to samples/bh-atomic/main.c diff --git a/samples/mem_allocator/CMakeLists.txt b/samples/mem-allocator/CMakeLists.txt similarity index 100% rename from samples/mem_allocator/CMakeLists.txt rename to samples/mem-allocator/CMakeLists.txt diff --git a/samples/mem_allocator/main.c b/samples/mem-allocator/main.c similarity index 100% rename from samples/mem_allocator/main.c rename to samples/mem-allocator/main.c From b45d01411200a6760bec790bf9db2c7c1ebdd8bb Mon Sep 17 00:00:00 2001 From: tonibofarull Date: Mon, 14 Aug 2023 14:03:56 +0200 Subject: [PATCH 14/34] wasi-nn: Improve TPU support (#2447) 1. Allow TPU and GPU support at the same time. 2. Add Dockerfile to run example with [Coral USB](https://coral.ai/products/accelerator/). --- build-scripts/config_common.cmake | 12 ++-- core/config.h | 8 +++ core/iwasm/libraries/wasi-nn/README.md | 30 +++++++--- .../wasi-nn/cmake/Findtensorflow_lite.cmake | 6 +- .../wasi-nn/src/wasi_nn_tensorflowlite.cpp | 59 +++++++++++++------ .../wasi-nn/test/Dockerfile.nvidia-gpu | 2 +- .../libraries/wasi-nn/test/Dockerfile.tpu | 37 ++++++++++++ core/iwasm/libraries/wasi-nn/test/utils.c | 4 +- core/iwasm/libraries/wasi-nn/test/utils.h | 2 +- doc/build_wamr.md | 7 ++- 10 files changed, 130 insertions(+), 37 deletions(-) create mode 100644 core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu diff --git a/build-scripts/config_common.cmake b/build-scripts/config_common.cmake index 73ed83c9..2751341e 100644 --- a/build-scripts/config_common.cmake +++ b/build-scripts/config_common.cmake @@ -359,16 +359,16 @@ endif () if (WAMR_BUILD_WASI_NN EQUAL 1) message (" WASI-NN enabled") add_definitions (-DWASM_ENABLE_WASI_NN=1) - if (WASI_NN_ENABLE_GPU EQUAL 1) + if (WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) message (" WASI-NN: GPU enabled") - add_definitions (-DWASI_NN_ENABLE_GPU=1) + add_definitions (-DWASM_ENABLE_WASI_NN_GPU=1) endif () - if (WAMR_BUILD_WASI_NN_ENABLE_EXT EQUAL 1) + if (WAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE EQUAL 1) message (" WASI-NN: External Delegation enabled") - add_definitions (-DWASI_NN_ENABLE_EXTERNAL_DELEGATE=1) + add_definitions (-DWASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE=1) endif () - if (DEFINED WASI_NN_EXT_DELEGATE_PATH) - add_definitions (-DWASI_NN_EXT_DELEGATE_PATH="${WASI_NN_EXT_DELEGATE_PATH}") + if (DEFINED WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH) + add_definitions (-DWASM_WASI_NN_EXTERNAL_DELEGATE_PATH="${WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH}") endif () endif () if (WAMR_BUILD_ALLOC_WITH_USER_DATA EQUAL 1) diff --git a/core/config.h b/core/config.h index b6198811..220f6751 100644 --- a/core/config.h +++ b/core/config.h @@ -144,6 +144,14 @@ #define WASM_ENABLE_WASI_NN 0 #endif +#ifndef WASM_ENABLE_WASI_NN_GPU +#define WASM_ENABLE_WASI_NN_GPU 0 +#endif + +#ifndef WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE +#define WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE 0 +#endif + /* Default disable libc emcc */ #ifndef WASM_ENABLE_LIBC_EMCC #define WASM_ENABLE_LIBC_EMCC 0 diff --git a/core/iwasm/libraries/wasi-nn/README.md b/core/iwasm/libraries/wasi-nn/README.md index 96493a30..fbc99ccb 100644 --- a/core/iwasm/libraries/wasi-nn/README.md +++ b/core/iwasm/libraries/wasi-nn/README.md @@ -25,6 +25,7 @@ Build the runtime image for your execution target type. * `cpu` * `nvidia-gpu` * `vx-delegate` +* `tpu` ``` EXECUTION_TYPE=cpu @@ -64,6 +65,8 @@ docker run \ ``` * (NVIDIA) GPU + * Requirements: + * [NVIDIA docker](https://github.com/NVIDIA/nvidia-docker). ``` docker run \ @@ -76,25 +79,36 @@ docker run \ /assets/test_tensorflow.wasm ``` -* vx-delegate for NPU (x86 simulater) +* vx-delegate for NPU (x86 simulator) ``` docker run \ - -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets wasi-nn-vx-delegate \ - --dir=/assets \ + -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets \ + wasi-nn-vx-delegate \ + --dir=/ \ --env="TARGET=gpu" \ - /assets/test_tensorflow.wasm + /assets/test_tensorflow_quantized.wasm ``` +* (Coral) TPU + * Requirements: + * [Coral USB](https://coral.ai/products/accelerator/). - -Requirements: -* [NVIDIA docker](https://github.com/NVIDIA/nvidia-docker). +``` +docker run \ + --privileged \ + --device=/dev/bus/usb:/dev/bus/usb \ + -v $PWD/core/iwasm/libraries/wasi-nn/test:/assets \ + wasi-nn-tpu \ + --dir=/ \ + --env="TARGET=tpu" \ + /assets/test_tensorflow_quantized.wasm +``` ## What is missing Supported: * Graph encoding: `tensorflowlite`. -* Execution target: `cpu` and `gpu`. +* Execution target: `cpu`, `gpu` and `tpu`. * Tensor type: `fp32`. diff --git a/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake b/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake index 39cc10dc..59fdaf75 100644 --- a/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake +++ b/core/iwasm/libraries/wasi-nn/cmake/Findtensorflow_lite.cmake @@ -18,12 +18,16 @@ if(NOT EXISTS ${TENSORFLOW_LITE}) set(TENSORFLOW_SOURCE_DIR "${WAMR_ROOT_DIR}/core/deps/tensorflow-src") - if(WASI_NN_ENABLE_GPU EQUAL 1) + if(WAMR_BUILD_WASI_NN_ENABLE_GPU EQUAL 1) # Tensorflow specific: # * https://www.tensorflow.org/lite/guide/build_cmake#available_options_to_build_tensorflow_lite set (TFLITE_ENABLE_GPU ON) endif() + if (CMAKE_SIZEOF_VOID_P EQUAL 4) + set (TFLITE_ENABLE_XNNPACK OFF) + endif() + add_subdirectory( "${TENSORFLOW_SOURCE_DIR}/tensorflow/lite" "${CMAKE_CURRENT_BINARY_DIR}/tensorflow-lite" diff --git a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp index b6fda1eb..65e38f86 100644 --- a/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp +++ b/core/iwasm/libraries/wasi-nn/src/wasi_nn_tensorflowlite.cpp @@ -16,11 +16,11 @@ #include #include -#if defined(WASI_NN_ENABLE_GPU) +#if WASM_ENABLE_WASI_NN_GPU != 0 #include #endif -#if defined(WASI_NN_ENABLE_EXTERNAL_DELEGATE) +#if WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE != 0 #include #endif @@ -130,8 +130,8 @@ tensorflowlite_load(void *tflite_ctx, graph_builder_array *builder, return invalid_argument; } - if (target != cpu && target != gpu) { - NN_ERR_PRINTF("Only CPU and GPU target is supported."); + if (target != cpu && target != gpu && target != tpu) { + NN_ERR_PRINTF("Only CPU, GPU and TPU target is supported."); return invalid_argument; } @@ -195,7 +195,7 @@ tensorflowlite_init_execution_context(void *tflite_ctx, graph g, switch (tfl_ctx->models[g].target) { case gpu: { -#if defined(WASI_NN_ENABLE_GPU) +#if WASM_ENABLE_WASI_NN_GPU != 0 NN_WARN_PRINTF("GPU enabled."); // https://www.tensorflow.org/lite/performance/gpu TfLiteGpuDelegateOptionsV2 options = @@ -216,10 +216,19 @@ tensorflowlite_init_execution_context(void *tflite_ctx, graph g, NN_ERR_PRINTF("Error when enabling GPU delegate."); use_default = true; } -#elif defined(WASI_NN_ENABLE_EXTERNAL_DELEGATE) +#else + NN_WARN_PRINTF("GPU not enabled."); + use_default = true; +#endif + break; + } + case tpu: + { +#if WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE != 0 NN_WARN_PRINTF("external delegation enabled."); TfLiteExternalDelegateOptions options = - TfLiteExternalDelegateOptionsDefault(WASI_NN_EXT_DELEGATE_PATH); + TfLiteExternalDelegateOptionsDefault( + WASM_WASI_NN_EXTERNAL_DELEGATE_PATH); tfl_ctx->delegate = TfLiteExternalDelegateCreate(&options); if (tfl_ctx->delegate == NULL) { NN_ERR_PRINTF("Error when generating External delegate."); @@ -233,7 +242,7 @@ tensorflowlite_init_execution_context(void *tflite_ctx, graph g, use_default = true; } #else - NN_WARN_PRINTF("GPU not enabled."); + NN_WARN_PRINTF("External delegate not enabled."); use_default = true; #endif break; @@ -438,19 +447,35 @@ tensorflowlite_destroy(void *tflite_ctx) */ TFLiteContext *tfl_ctx = (TFLiteContext *)tflite_ctx; - if (tfl_ctx->delegate != NULL) { -#if defined(WASI_NN_ENABLE_GPU) - TfLiteGpuDelegateV2Delete(tfl_ctx->delegate); -#elif defined(WASI_NN_ENABLE_EXTERNAL_DELEGATE) - TfLiteExternalDelegateDelete(tfl_ctx->delegate); -#endif - } - NN_DBG_PRINTF("Freeing memory."); for (int i = 0; i < MAX_GRAPHS_PER_INST; ++i) { tfl_ctx->models[i].model.reset(); - if (tfl_ctx->models[i].model_pointer) + if (tfl_ctx->models[i].model_pointer) { + if (tfl_ctx->delegate) { + switch (tfl_ctx->models[i].target) { + case gpu: + { +#if WASM_ENABLE_WASI_NN_GPU != 0 + TfLiteGpuDelegateV2Delete(tfl_ctx->delegate); +#else + NN_ERR_PRINTF("GPU delegate delete but not enabled."); +#endif + break; + } + case tpu: + { +#if WASM_ENABLE_WASI_NN_EXTERNAL_DELEGATE != 0 + TfLiteExternalDelegateDelete(tfl_ctx->delegate); +#else + NN_ERR_PRINTF( + "External delegate delete but not enabled."); +#endif + break; + } + } + } wasm_runtime_free(tfl_ctx->models[i].model_pointer); + } tfl_ctx->models[i].model_pointer = NULL; } for (int i = 0; i < MAX_GRAPH_EXEC_CONTEXTS_PER_INST; ++i) { diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu index d78021ea..737b2d3f 100644 --- a/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.nvidia-gpu @@ -24,7 +24,7 @@ RUN apt-get install -y wget ca-certificates --no-install-recommends \ RUN cmake \ -DWAMR_BUILD_WASI_NN=1 \ - -DWASI_NN_ENABLE_GPU=1 \ + -DWAMR_BUILD_WASI_NN_ENABLE_GPU=1 \ .. RUN make -j "$(grep -c ^processor /proc/cpuinfo)" diff --git a/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu b/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu new file mode 100644 index 00000000..742645b0 --- /dev/null +++ b/core/iwasm/libraries/wasi-nn/test/Dockerfile.tpu @@ -0,0 +1,37 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +FROM ubuntu:20.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y \ + cmake build-essential git curl gnupg --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# hadolint ignore=DL3008,DL4006 +RUN echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | tee /etc/apt/sources.list.d/coral-edgetpu.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ + apt-get update && apt-get install -y libedgetpu1-std --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /home/wamr + +COPY . . + +WORKDIR /home/wamr/product-mini/platforms/linux/build + +RUN cmake \ + -DWAMR_BUILD_WASI_NN=1 \ + -DWAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE=1 \ + -DWAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH="libedgetpu.so.1.0" \ + -DWAMR_BUILD_WASI_NN_ENABLE_GPU=1 \ + .. + +RUN make -j "$(grep -c ^processor /proc/cpuinfo)" && \ + cp /home/wamr/product-mini/platforms/linux/build/iwasm /iwasm + +WORKDIR /assets + +ENTRYPOINT [ "/iwasm" ] diff --git a/core/iwasm/libraries/wasi-nn/test/utils.c b/core/iwasm/libraries/wasi-nn/test/utils.c index e0704cab..7b4f65d1 100644 --- a/core/iwasm/libraries/wasi-nn/test/utils.c +++ b/core/iwasm/libraries/wasi-nn/test/utils.c @@ -132,8 +132,8 @@ run_inference(execution_target target, float *input, uint32_t *input_size, *output_size = MAX_OUTPUT_TENSOR_SIZE - *output_size; if (wasm_get_output(ctx, i, &out_tensor[offset], output_size) != success) { - NN_ERR_PRINTF("Error when getting output."); - exit(1); + NN_ERR_PRINTF("Error when getting index %d.", i); + break; } offset += *output_size; diff --git a/core/iwasm/libraries/wasi-nn/test/utils.h b/core/iwasm/libraries/wasi-nn/test/utils.h index 6373be54..0b232840 100644 --- a/core/iwasm/libraries/wasi-nn/test/utils.h +++ b/core/iwasm/libraries/wasi-nn/test/utils.h @@ -11,7 +11,7 @@ #include "wasi_nn.h" #define MAX_MODEL_SIZE 85000000 -#define MAX_OUTPUT_TENSOR_SIZE 200 +#define MAX_OUTPUT_TENSOR_SIZE 1000000 #define INPUT_TENSOR_DIMS 4 #define EPSILON 1e-8 diff --git a/doc/build_wamr.md b/doc/build_wamr.md index a9d33716..81b4f621 100644 --- a/doc/build_wamr.md +++ b/doc/build_wamr.md @@ -94,7 +94,12 @@ cmake -DWAMR_BUILD_PLATFORM=linux -DWAMR_BUILD_TARGET=ARM - **WAMR_BUILD_WASI_NN**=1/0, default to disable if not set #### **Enable lib wasi-nn GPU mode** -- **WASI_NN_ENABLE_GPU**=1/0, default to disable if not set +- **WAMR_BUILD_WASI_NN_ENABLE_GPU**=1/0, default to disable if not set + +#### **Enable lib wasi-nn external delegate mode** +- **WAMR_BUILD_WASI_NN_ENABLE_EXTERNAL_DELEGATE**=1/0, default to disable if not set + +- **WAMR_BUILD_WASI_NN_EXTERNAL_DELEGATE_PATH**=Path to the external delegate shared library (e.g. `libedgetpu.so.1.0` for Coral USB) #### **Disable boundary check with hardware trap** - **WAMR_DISABLE_HW_BOUND_CHECK**=1/0, default to enable if not set and supported by platform From 365cdfeb712c10c8925e6befe73544de9a4cb597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A4mes=20M=C3=A9n=C3=A9trey?= Date: Tue, 15 Aug 2023 04:12:43 +0200 Subject: [PATCH 15/34] Upgrade SGX-RA integration for 0.1.2 and Ubuntu 20.04 (#2454) --- core/iwasm/libraries/lib-rats/lib_rats.cmake | 10 +++++ .../linux-sgx/enclave-sample/Makefile | 6 +-- samples/sgx-ra/README.md | 43 +++++++++---------- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/core/iwasm/libraries/lib-rats/lib_rats.cmake b/core/iwasm/libraries/lib-rats/lib_rats.cmake index b773c837..3ca08f16 100644 --- a/core/iwasm/libraries/lib-rats/lib_rats.cmake +++ b/core/iwasm/libraries/lib-rats/lib_rats.cmake @@ -23,6 +23,7 @@ include(FetchContent) set(RATS_BUILD_MODE "sgx" CACHE INTERNAL "Select build mode for librats(host|occlum|sgx|wasm)") set(RATS_INSTALL_PATH "${CMAKE_BINARY_DIR}/librats" CACHE INTERNAL "") +set(BUILD_SAMPLES OFF) FetchContent_Declare( librats @@ -34,8 +35,17 @@ if (NOT librats_POPULATED) message("-- Fetching librats ..") FetchContent_Populate(librats) include_directories("${librats_SOURCE_DIR}/include") + + # Prevent the propagation of the CMAKE_C_FLAGS of WAMR into librats + set(SAVED_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + set(CMAKE_C_FLAGS "") + + # Import the building scripts of librats add_subdirectory(${librats_SOURCE_DIR} ${librats_BINARY_DIR} EXCLUDE_FROM_ALL) + # Restore the CMAKE_C_FLAGS of WAMR + set(CMAKE_C_FLAGS ${SAVED_CMAKE_C_FLAGS}) + endif() file (GLOB source_all ${LIB_RATS_DIR}/*.c) diff --git a/product-mini/platforms/linux-sgx/enclave-sample/Makefile b/product-mini/platforms/linux-sgx/enclave-sample/Makefile index 40254562..dae97a06 100644 --- a/product-mini/platforms/linux-sgx/enclave-sample/Makefile +++ b/product-mini/platforms/linux-sgx/enclave-sample/Makefile @@ -143,10 +143,10 @@ else endif ifeq ($(WAMR_BUILD_LIB_RATS), 1) - Rats_Lib_Link_Dirs := -L$(LIB_RATS_INSTALL_DIR) -L$(LIB_RATS_INSTALL_DIR)/attesters -L$(LIB_RATS_INSTALL_DIR)/verifiers -L$(SGX_SSL)/lib64 + Rats_Lib_Link_Dirs := -L$(LIB_RATS_INSTALL_DIR) -L$(LIB_RATS_INSTALL_DIR)/attesters -L$(LIB_RATS_INSTALL_DIR)/verifiers -L$(SGX_SSL)/lib64 -L$(VMLIB_BUILD_DIR)/external/libcbor/src/libcbor/lib -L$(LIB_RATS_INSTALL_DIR)/crypto_wrappers Rats_Lib_W_Link_libs := -lattester_nullattester -lattester_sgx_ecdsa -lattester_sgx_la \ - -lverifier_nullverifier -lverifier_sgx_ecdsa -lverifier_sgx_la -lverifier_sgx_ecdsa_qve \ - -lrats_lib -lsgx_tsgxssl + -lverifier_nullverifier -lverifier_sgx_la -lverifier_sgx_ecdsa_qve -lcbor \ + -lrats_lib -lsgx_tsgxssl -lcrypto_wrapper_nullcrypto -lcrypto_wrapper_openssl Rats_Lib_NW_Link_libs := -lsgx_dcap_tvl -lsgx_tsgxssl_crypto endif diff --git a/samples/sgx-ra/README.md b/samples/sgx-ra/README.md index 39a2f2d9..9270c96f 100644 --- a/samples/sgx-ra/README.md +++ b/samples/sgx-ra/README.md @@ -7,52 +7,52 @@ This sample demonstrates how to execute Remote Attestation on SGX with [librats] SGX-RA requires to have installed: - the WASI-SDK, located in `/opt/wasi-sdk` - - CMake >= 3.11, which is not provided on Ubuntu 18.04 (use [Kitware APT Repository](https://apt.kitware.com/)) ### Intel SGX dependencies Before starting, we need to download and install [SGX SDK](https://download.01.org/intel-sgx/latest/linux-latest/distro) and [SGX DCAP Library](https://download.01.org/intel-sgx/latest/dcap-latest) referring to this [guide](https://download.01.org/intel-sgx/sgx-dcap/1.8/linux/docs/Intel_SGX_DCAP_Linux_SW_Installation_Guide.pdf). -The following commands are an example of the SGX environment installation on Ubuntu 18.04. +The following commands are an example of the SGX environment installation on Ubuntu 20.04. ``` shell # Set your platform, you can get the platforms list on # https://download.01.org/intel-sgx/latest/linux-latest/distro $ cd $HOME -$ SGX_PLATFORM=ubuntu18.04-server -$ SGX_SDK_VERSION=2.17.100.3 +$ OS_PLATFORM=ubuntu20.04 +$ SGX_PLATFORM=$OS_PLATFORM-server +$ SGX_RELEASE_VERSION=1.17 $ SGX_DRIVER_VERSION=1.41 +$ SGX_SDK_VERSION=2.20.100.4 # install the dependencies $ sudo apt-get update -$ sudo apt-get install -y dkms +$ sudo apt-get install -y build-essential ocaml automake autoconf libtool wget python3 libssl-dev dkms zip cmake +$ sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1 # install SGX Driver -$ wget https://download.01.org/intel-sgx/latest/linux-latest/distro/$SGX_PLATFORM/sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin +$ wget https://download.01.org/intel-sgx/sgx-dcap/$SGX_RELEASE_VERSION/linux/distro/$SGX_PLATFORM/sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin $ chmod +x sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin $ sudo ./sgx_linux_x64_driver_$SGX_DRIVER_VERSION.bin # install SGX SDK -$ wget https://download.01.org/intel-sgx/latest/linux-latest/distro/$SGX_PLATFORM/sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin +$ wget https://download.01.org/intel-sgx/sgx-dcap/$SGX_RELEASE_VERSION/linux/distro/$SGX_PLATFORM/sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin $ chmod +x sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin -$ sudo ./sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin +$ sudo ./sgx_linux_x64_sdk_$SGX_SDK_VERSION.bin --prefix /opt/intel # install SGX DCAP Library -$ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list > /dev/null -$ wget -O - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add - +$ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | sudo tee /etc/apt/sources.list.d/intel-sgx.list +$ wget -O - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | sudo apt-key add $ sudo apt-get update -$ sudo apt-get install -y libsgx-uae-service libsgx-dcap-default-qpl-dev libsgx-dcap-ql-dev libsgx-dcap-quote-verify-dev +$ sudo apt-get install -y libsgx-epid libsgx-quote-ex libsgx-dcap-ql libsgx-enclave-common-dev libsgx-dcap-ql-dev libsgx-dcap-default-qpl-dev libsgx-dcap-quote-verify-dev # install SGX SSL Library $ git clone https://github.com/intel/linux-sgx.git $ cd linux-sgx && make preparation -$ sudo cp external/toolset/{current_distr}/* /usr/local/bin +$ sudo cp external/toolset/$OS_PLATFORM/* /usr/local/bin $ # Verify that the paths are correctly set $ which ar as ld objcopy objdump ranlib $ cd ../ $ git clone https://github.com/intel/intel-sgx-ssl.git -$ wget https://www.openssl.org/source/openssl-1.1.1q.tar.gz -$ cp openssl-1.1.1q.tar.gz intel-sgx-ssl/openssl_source -$ rm -f openssl-1.1.1q.tar.gz +$ wget https://www.openssl.org/source/openssl-1.1.1v.tar.gz -O intel-sgx-ssl/openssl_source/openssl-1.1.1v.tar.gz $ cd intel-sgx-ssl/Linux $ source /opt/intel/sgxsdk/environment $ make all @@ -70,7 +70,7 @@ sudo usermod -a -G sgx_prv Intel DCAP connects to Intel PCS to download the attestation collateral for SGX-enabled machines. Intel provides a [quick install guide](https://www.intel.com/content/www/us/en/developer/articles/guide/intel-software-guard-extensions-data-center-attestation-primitives-quick-install-guide.html) to set up a simplified environment. -This section summarizes the commands to issue for setting up a working environment on Ubuntu 18.04. +This section summarizes the commands to issue for setting up a working environment on Ubuntu 20.04. ### Subscribe to Intel PCS Web services @@ -86,11 +86,9 @@ Intel provides an implementation of the cache mechanism. The following commands set up Intel PCCS. ```shell # install Node.js -$ curl -o setup.sh -sL https://deb.nodesource.com/setup_14.x -$ chmod a+x setup.sh -$ sudo ./setup.sh +$ curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get install -y nodejs # install PCCS software -$ sudo apt-get install -y cracklib-runtime sqlite3 python build-essential +$ sudo apt-get install -y cracklib-runtime $ sudo apt-get install -y sgx-dcap-pccs ``` @@ -160,9 +158,10 @@ Adapt the configuration file of `PCKIDRetrievalTool` located in `/opt/intel/sgx- Save your changes and run the provisioning tool. ```shell -$ PCKIDRetrievalTool -Intel(R) Software Guard Extensions PCK Cert ID Retrieval Tool Version 1.14.100.3 +$ sudo PCKIDRetrievalTool +Intel(R) Software Guard Extensions PCK Cert ID Retrieval Tool Version 1.17.100.4 +Registration status has been set to completed status. the data has been sent to cache server successfully and pckid_retrieval.csv has been generated successfully! ``` From 571c057549a39664403c8627ec9c04a34f6c4451 Mon Sep 17 00:00:00 2001 From: tonibofarull Date: Tue, 15 Aug 2023 04:32:43 +0200 Subject: [PATCH 16/34] wamr-python: Enable debugging WASM and grant dir access (#2449) - Enable debugging a WASM loaded and executed from Python. - Expose API to enable access to list of host directories. Similar to --dir in iwasm. - Add another python language binding sample: native-symbol. --- .../python/src/wamr/wamrapi/wamr.py | 143 +++++++++++++++--- language-bindings/python/utils/create_lib.sh | 7 +- language-bindings/python/wamr-api/README.md | 9 +- .../wamr-api/samples/{ => basic}/compile.sh | 0 .../wamr-api/samples/{ => basic}/main.py | 0 .../python/wamr-api/samples/{ => basic}/sum.c | 0 .../wamr-api/samples/native-symbol/README.md | 44 ++++++ .../wamr-api/samples/native-symbol/compile.sh | 14 ++ .../wamr-api/samples/native-symbol/func.c | 30 ++++ .../wamr-api/samples/native-symbol/main.py | 59 ++++++++ 10 files changed, 277 insertions(+), 29 deletions(-) rename language-bindings/python/wamr-api/samples/{ => basic}/compile.sh (100%) rename language-bindings/python/wamr-api/samples/{ => basic}/main.py (100%) rename language-bindings/python/wamr-api/samples/{ => basic}/sum.c (100%) create mode 100644 language-bindings/python/wamr-api/samples/native-symbol/README.md create mode 100755 language-bindings/python/wamr-api/samples/native-symbol/compile.sh create mode 100644 language-bindings/python/wamr-api/samples/native-symbol/func.c create mode 100644 language-bindings/python/wamr-api/samples/native-symbol/main.py diff --git a/language-bindings/python/src/wamr/wamrapi/wamr.py b/language-bindings/python/src/wamr/wamrapi/wamr.py index abbd2322..74cbad9f 100644 --- a/language-bindings/python/src/wamr/wamrapi/wamr.py +++ b/language-bindings/python/src/wamr/wamrapi/wamr.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception from ctypes import Array +from ctypes import addressof from ctypes import c_char from ctypes import c_uint from ctypes import c_uint8 @@ -10,6 +11,8 @@ from ctypes import cast from ctypes import create_string_buffer from ctypes import POINTER from ctypes import pointer +from typing import List +from typing import Tuple from wamr.wamrapi.iwasm import String from wamr.wamrapi.iwasm import Alloc_With_Pool from wamr.wamrapi.iwasm import RuntimeInitArgs @@ -31,6 +34,14 @@ from wamr.wamrapi.iwasm import wasm_runtime_module_malloc from wamr.wamrapi.iwasm import wasm_runtime_module_free from wamr.wamrapi.iwasm import wasm_runtime_register_natives from wamr.wamrapi.iwasm import NativeSymbol +from wamr.wamrapi.iwasm import wasm_runtime_start_debug_instance +from wamr.wamrapi.iwasm import wasm_runtime_call_indirect +from wamr.wamrapi.iwasm import wasm_runtime_get_module_inst +from wamr.wamrapi.iwasm import wasm_runtime_addr_app_to_native +from wamr.wamrapi.iwasm import wasm_runtime_addr_native_to_app +from wamr.wamrapi.iwasm import wasm_runtime_set_wasi_args + +ID_TO_EXEC_ENV_MAPPING = {} class Engine: @@ -43,16 +54,26 @@ class Engine: print("deleting Engine") wasm_runtime_destroy() - def _get_init_args(self, heap_size: int = 1024 * 512) -> RuntimeInitArgs: + def _get_init_args( + self, + heap_size: int = 1024 * 1024 * 2, + ip_addr: str = "127.0.0.1", + instance_port: int = 1234, + ) -> RuntimeInitArgs: init_args = RuntimeInitArgs() init_args.mem_alloc_type = Alloc_With_Pool init_args.mem_alloc_option.pool.heap_buf = cast( (c_char * heap_size)(), c_void_p ) init_args.mem_alloc_option.pool.heap_size = heap_size + # Debug port setting + init_args.ip_addr = bytes(ip_addr, "utf-8") + init_args.instance_port = instance_port return init_args - def register_natives(self, module_name: str, native_symbols: list[NativeSymbol]) -> None: + def register_natives( + self, module_name: str, native_symbols: List[NativeSymbol] + ) -> None: module_name = String.from_param(module_name) # WAMR does not copy the symbols. We must store them. for native in native_symbols: @@ -62,12 +83,13 @@ class Engine: module_name, cast( (NativeSymbol * len(native_symbols))(*native_symbols), - POINTER(NativeSymbol) + POINTER(NativeSymbol), ), - len(native_symbols) + len(native_symbols), ): raise Exception("Error while registering symbols") + class Module: __create_key = object() @@ -86,7 +108,7 @@ class Module: print("deleting Module") wasm_runtime_unload(self.module) - def _create_module(self, fp: str) -> tuple[wasm_module_t, Array[c_uint]]: + def _create_module(self, fp: str) -> Tuple[wasm_module_t, "Array[c_uint]"]: with open(fp, "rb") as f: data = f.read() data = (c_uint8 * len(data))(*data) @@ -99,14 +121,52 @@ class Module: class Instance: - def __init__(self, module: Module, stack_size: int = 65536, heap_size: int = 16384): + def __init__( + self, + module: Module, + stack_size: int = 65536, + heap_size: int = 16384, + dir_list: List[str] | None = None, + preinitialized_module_inst: wasm_module_inst_t | None = None, + ): + # Store module ensures GC does not remove it self.module = module - self.module_inst = self._create_module_inst(module, stack_size, heap_size) + if preinitialized_module_inst is None: + self.module_inst = self._create_module_inst(module, stack_size, heap_size) + else: + self.module_inst = preinitialized_module_inst + if dir_list: + self._set_wasi_args(module, dir_list) def __del__(self): print("deleting Instance") wasm_runtime_deinstantiate(self.module_inst) + def _set_wasi_args(self, module: Module, dir_list: List[str]) -> None: + LP_c_char = POINTER(c_char) + LP_LP_c_char = POINTER(LP_c_char) + + p = (LP_c_char * len(dir_list))() + for i, dir in enumerate(dir_list): + enc_dir = dir.encode("utf-8") + p[i] = create_string_buffer(enc_dir) + + na = cast(p, LP_LP_c_char) + wasm_runtime_set_wasi_args( + module.module, na, len(dir_list), None, 0, None, 0, None, 0 + ) + + def _create_module_inst( + self, module: Module, stack_size: int, heap_size: int + ) -> wasm_module_inst_t: + error_buf = create_string_buffer(128) + module_inst = wasm_runtime_instantiate( + module.module, stack_size, heap_size, error_buf, len(error_buf) + ) + if not module_inst: + raise Exception("Error while creating module instance") + return module_inst + def malloc(self, nbytes: int, native_handler) -> c_uint: return wasm_runtime_module_malloc(self.module_inst, nbytes, native_handler) @@ -119,31 +179,70 @@ class Instance: raise Exception("Error while looking-up function") return func - def _create_module_inst(self, module: Module, stack_size: int, heap_size: int) -> wasm_module_inst_t: - error_buf = create_string_buffer(128) - module_inst = wasm_runtime_instantiate( - module.module, stack_size, heap_size, error_buf, len(error_buf) - ) - if not module_inst: - raise Exception("Error while creating module instance") - return module_inst + def native_addr_to_app_addr(self, native_addr) -> c_void_p: + return wasm_runtime_addr_native_to_app(self.module_inst, native_addr) + + def app_addr_to_native_addr(self, app_addr) -> c_void_p: + return wasm_runtime_addr_app_to_native(self.module_inst, app_addr) class ExecEnv: def __init__(self, module_inst: Instance, stack_size: int = 65536): self.module_inst = module_inst self.exec_env = self._create_exec_env(module_inst, stack_size) + self.env = addressof(self.exec_env.contents) + self.own_c = True + + ID_TO_EXEC_ENV_MAPPING[str(self.env)] = self def __del__(self): - print("deleting ExecEnv") - wasm_runtime_destroy_exec_env(self.exec_env) + if self.own_c: + print("deleting ExecEnv") + wasm_runtime_destroy_exec_env(self.exec_env) + del ID_TO_EXEC_ENV_MAPPING[str(self.env)] + + def _create_exec_env( + self, module_inst: Instance, stack_size: int + ) -> wasm_exec_env_t: + exec_env = wasm_runtime_create_exec_env(module_inst.module_inst, stack_size) + if not exec_env: + raise Exception("Error while creating execution environment") + return exec_env def call(self, func: wasm_function_inst_t, argc: int, argv: "POINTER[c_uint]"): if not wasm_runtime_call_wasm(self.exec_env, func, argc, argv): raise Exception("Error while calling function") - def _create_exec_env(self, module_inst: Instance, stack_size: int) -> wasm_exec_env_t: - exec_env = wasm_runtime_create_exec_env(module_inst.module_inst, stack_size) - if not exec_env: - raise Exception("Error while creating execution environment") - return exec_env + def get_module_inst(self) -> Instance: + return self.module_inst + + def start_debugging(self) -> int: + return wasm_runtime_start_debug_instance(self.exec_env) + + def call_indirect(self, element_index: int, argc: int, argv: "POINTER[c_uint]"): + if not wasm_runtime_call_indirect(self.exec_env, element_index, argc, argv): + raise Exception("Error while calling function") + + @staticmethod + def wrap(env: int) -> "ExecEnv": + if str(env) in ID_TO_EXEC_ENV_MAPPING: + return ID_TO_EXEC_ENV_MAPPING[str(env)] + return InternalExecEnv(env) + + +class InternalExecEnv(ExecEnv): + """ + Generate Python ExecEnv-like object from a `wasm_exec_env_t` index. + """ + + def __init__(self, env: int): + self.env = env + self.exec_env = cast(env, wasm_exec_env_t) + self.module_inst = Instance( + module=object(), + preinitialized_module_inst=wasm_runtime_get_module_inst(self.exec_env), + ) + ID_TO_EXEC_ENV_MAPPING[str(env)] = self + + def __del__(self): + del ID_TO_EXEC_ENV_MAPPING[str(self.env)] diff --git a/language-bindings/python/utils/create_lib.sh b/language-bindings/python/utils/create_lib.sh index b7e10d3e..801186e9 100755 --- a/language-bindings/python/utils/create_lib.sh +++ b/language-bindings/python/utils/create_lib.sh @@ -12,7 +12,12 @@ WAMR_BUILD_PLATFORM=${WAMR_BUILD_PLATFORM:-${UNAME}} cd ${ROOT_DIR}/product-mini/platforms/${WAMR_BUILD_PLATFORM} mkdir -p build && cd build -cmake .. +cmake \ + -DWAMR_BUILD_DEBUG_INTERP=1 \ + -DWAMR_BUILD_LIB_PTHREAD=1 \ + -DWAMR_BUILD_LIB_WASI_THREADS=1 \ + -DWAMR_BUILD_LIB_WASI=1 \ + .. make -j case ${UNAME} in diff --git a/language-bindings/python/wamr-api/README.md b/language-bindings/python/wamr-api/README.md index 801c8aa1..236150ce 100644 --- a/language-bindings/python/wamr-api/README.md +++ b/language-bindings/python/wamr-api/README.md @@ -22,10 +22,7 @@ bash language-bindings/python/utils/create_lib.sh This will build and copy libiwasm into the package. -## Examples +## Samples -There is a [simple example](./samples/main.py) to show how to use bindings. - -``` -python samples/main.py -``` +- **[basic](./samples/basic)**: Demonstrating how to use basic python bindings. +- **[native-symbol](./samples/native-symbol)**: Desmostrate how to call WASM from Python and how to export Python functions into WASM. diff --git a/language-bindings/python/wamr-api/samples/compile.sh b/language-bindings/python/wamr-api/samples/basic/compile.sh similarity index 100% rename from language-bindings/python/wamr-api/samples/compile.sh rename to language-bindings/python/wamr-api/samples/basic/compile.sh diff --git a/language-bindings/python/wamr-api/samples/main.py b/language-bindings/python/wamr-api/samples/basic/main.py similarity index 100% rename from language-bindings/python/wamr-api/samples/main.py rename to language-bindings/python/wamr-api/samples/basic/main.py diff --git a/language-bindings/python/wamr-api/samples/sum.c b/language-bindings/python/wamr-api/samples/basic/sum.c similarity index 100% rename from language-bindings/python/wamr-api/samples/sum.c rename to language-bindings/python/wamr-api/samples/basic/sum.c diff --git a/language-bindings/python/wamr-api/samples/native-symbol/README.md b/language-bindings/python/wamr-api/samples/native-symbol/README.md new file mode 100644 index 00000000..75d4fab4 --- /dev/null +++ b/language-bindings/python/wamr-api/samples/native-symbol/README.md @@ -0,0 +1,44 @@ +# Native Symbol + +This sample demonstrates how to declare a Python function as `NativeSymbol`. + +Steps of the example: +1. Load WASM from Python +2. Call `c_func` from WASM. +3. `c_func` calls `python_func` from Python. +4. `python_func` calls `add` from WASM. +5. Result shown by Python. + +## Build + +Follow instructions [build wamr Python package](../../README.md). + +Compile WASM app example, + +```sh +./compile.sh +``` + +## Run sample + +```sh +python main.py +``` + +Output: + +``` +python: calling c_func(10) +c: in c_func with input: 10 +c: calling python_func(11) +python: in python_func with input: 11 +python: calling add(11, 1000) +python: result from add: 1011 +c: result from python_func: 1012 +c: returning 1013 +python: result from c_func: 1013 +deleting ExecEnv +deleting Instance +deleting Module +deleting Engine +``` diff --git a/language-bindings/python/wamr-api/samples/native-symbol/compile.sh b/language-bindings/python/wamr-api/samples/native-symbol/compile.sh new file mode 100755 index 00000000..4e504b01 --- /dev/null +++ b/language-bindings/python/wamr-api/samples/native-symbol/compile.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +/opt/wasi-sdk/bin/clang \ + -O0 -z stack-size=4096 -Wl,--initial-memory=65536 \ + -Wl,--export=main -Wl,--export=__main_argc_argv \ + -Wl,--export=__data_end -Wl,--export=__heap_base \ + -Wl,--strip-all,--no-entry \ + -Wl,--allow-undefined \ + -Wl,--export=c_func\ + -Wl,--export=add\ + -o func.wasm func.c diff --git a/language-bindings/python/wamr-api/samples/native-symbol/func.c b/language-bindings/python/wamr-api/samples/native-symbol/func.c new file mode 100644 index 00000000..4411a6b2 --- /dev/null +++ b/language-bindings/python/wamr-api/samples/native-symbol/func.c @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 Intel Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include + +int +python_func(int val); + +int +add(int val1, int val2) +{ + return val1 + val2; +} + +int +c_func(int val) +{ + printf("c: in c_func with input: %d\n", val); + printf("c: calling python_func(%d)\n", val + 1); + int res = python_func(val + 1); + printf("c: result from python_func: %d\n", res); + printf("c: returning %d\n", res + 1); + return res + 1; +} + +int +main() +{} diff --git a/language-bindings/python/wamr-api/samples/native-symbol/main.py b/language-bindings/python/wamr-api/samples/native-symbol/main.py new file mode 100644 index 00000000..3871ef13 --- /dev/null +++ b/language-bindings/python/wamr-api/samples/native-symbol/main.py @@ -0,0 +1,59 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +from wamr.wamrapi.wamr import Engine, Module, Instance, ExecEnv +from ctypes import c_uint +import pathlib +from ctypes import c_int32 +from ctypes import c_uint +from ctypes import c_void_p +from ctypes import cast +from ctypes import CFUNCTYPE + +from wamr.wamrapi.iwasm import NativeSymbol +from wamr.wamrapi.iwasm import String +from wamr.wamrapi.wamr import ExecEnv + +def python_func(env: int, value: int) -> int: + print("python: in python_func with input:", value) + # Example of generating ExecEnv from `wasm_exec_env_t`` + exec_env = ExecEnv.wrap(env) + add = exec_env.get_module_inst().lookup_function("add") + const = 1000 + argv = (c_uint * 2)(value, const) + print(f"python: calling add({value}, {const})") + exec_env.call(add, 2, argv) + res = argv[0] + print("python: result from add:", res) + return res + 1 + + +native_symbols = (NativeSymbol * 1)( + *[ + NativeSymbol( + symbol=String.from_param("python_func"), + func_ptr=cast( + CFUNCTYPE(c_int32, c_void_p, c_int32)(python_func), c_void_p + ), + signature=String.from_param("(i)i"), + ) + ] +) + +def main(): + engine = Engine() + engine.register_natives("env", native_symbols) + module = Module.from_file(engine, pathlib.Path(__file__).parent / "func.wasm") + module_inst = Instance(module) + exec_env = ExecEnv(module_inst) + + func = module_inst.lookup_function("c_func") + + inp = 10 + print(f"python: calling c_func({inp})") + argv = (c_uint)(inp) + exec_env.call(func, 1, argv) + print("python: result from c_func:", argv.value) + +if __name__ == "__main__": + main() From a9bd6af0ac70a7e9c634917e8ff1d3a2e475a47e Mon Sep 17 00:00:00 2001 From: Maks Litskevich Date: Tue, 15 Aug 2023 03:36:10 +0100 Subject: [PATCH 17/34] Build wasi-libc from source in WAMR CI (#2465) We need to apply some bug fixes that were merged to wasi-libc because wasi-sdk-20 is about half a year old. It is a temporary solution and the code will be removed when wasi-sdk 21 is released. --- .../compilation_on_android_ubuntu.yml | 12 ++++++- .github/workflows/nightly_run.yml | 12 ++++++- .../libraries/lib-wasi-threads/test/build.sh | 33 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index 5a0939b7..ee22af86 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -483,6 +483,16 @@ jobs: sudo tar -xzf wasi-sdk-*.tar.gz sudo mv wasi-sdk-20.0 wasi-sdk + # It is a temporary solution until new wasi-sdk that includes bug fixes is released + - name: build wasi-libc from source + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + git clone https://github.com/WebAssembly/wasi-libc + cd wasi-libc + make -j AR=/opt/wasi-sdk/bin/llvm-ar NM=/opt/wasi-sdk/bin/llvm-nm CC=/opt/wasi-sdk/bin/clang THREAD_MODEL=posix + echo "SYSROOT_PATH=$PWD/sysroot" >> $GITHUB_ENV + + - name: set env variable(if llvm are used) if: matrix.running_mode == 'aot' || matrix.running_mode == 'jit' || matrix.running_mode == 'multi-tier-jit' run: echo "USE_LLVM=true" >> $GITHUB_ENV @@ -518,7 +528,7 @@ jobs: - name: Build WASI thread tests if: matrix.test_option == '$WASI_TEST_OPTIONS' - run: bash build.sh + run: bash build.sh --sysroot "$SYSROOT_PATH" working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/ - name: build socket api tests diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index cea5dccb..b6cc15dc 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -548,6 +548,16 @@ jobs: sudo wget ${{ matrix.wasi_sdk_release }} sudo tar -xzf wasi-sdk-*.tar.gz sudo mv wasi-sdk-20.0 wasi-sdk + + # It is a temporary solution until new wasi-sdk that includes bug fixes is released + - name: build wasi-libc from source + if: matrix.test_option == '$WASI_TEST_OPTIONS' + run: | + git clone https://github.com/WebAssembly/wasi-libc + cd wasi-libc + make -j AR=/opt/wasi-sdk/bin/llvm-ar NM=/opt/wasi-sdk/bin/llvm-nm CC=/opt/wasi-sdk/bin/clang THREAD_MODEL=posix + echo "SYSROOT_PATH=$PWD/sysroot" >> $GITHUB_ENV + - name: set env variable(if llvm are used) if: matrix.running_mode == 'aot' || matrix.running_mode == 'jit' || matrix.running_mode == 'multi-tier-jit' run: echo "USE_LLVM=true" >> $GITHUB_ENV @@ -586,7 +596,7 @@ jobs: - name: Build WASI thread tests if: matrix.test_option == '$WASI_TEST_OPTIONS' - run: bash build.sh + run: bash build.sh --sysroot "$SYSROOT_PATH" working-directory: ./core/iwasm/libraries/lib-wasi-threads/test/ - name: build socket api tests diff --git a/core/iwasm/libraries/lib-wasi-threads/test/build.sh b/core/iwasm/libraries/lib-wasi-threads/test/build.sh index 611a3bd4..7c9a773c 100755 --- a/core/iwasm/libraries/lib-wasi-threads/test/build.sh +++ b/core/iwasm/libraries/lib-wasi-threads/test/build.sh @@ -9,6 +9,30 @@ set -eo pipefail CC=${CC:=/opt/wasi-sdk/bin/clang} WAMR_DIR=../../../../.. +show_usage() { + echo "Usage: $0 [--sysroot PATH_TO_SYSROOT]" + echo "--sysroot PATH_TO_SYSROOT specify to build with custom sysroot for wasi-libc" +} + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --sysroot) + sysroot_path="$2" + shift + shift + ;; + --help) + show_usage + exit + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + # Stress tests names thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm") @@ -21,6 +45,14 @@ for test_c in *.c; do thread_start_file=$WAMR_DIR/samples/wasi-threads/wasm-apps/wasi_thread_start.S fi + if [[ -n "$sysroot_path" ]]; then + if [ ! -d "$sysroot_path" ]; then + echo "Directory $sysroot_path doesn't exist. Aborting" + exit 1 + fi + sysroot_command="--sysroot $sysroot_path" + fi + echo "Compiling $test_c to $test_wasm" $CC \ -target wasm32-wasi-threads \ @@ -33,6 +65,7 @@ for test_c in *.c; do -Wl,--export=malloc \ -Wl,--export=free \ -I $WAMR_DIR/samples/wasi-threads/wasm-apps \ + $sysroot_command \ $thread_start_file \ $test_c -o $test_wasm done \ No newline at end of file From 0f18051e6695b575070eaa7541fe3bb9e097c613 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 15 Aug 2023 12:15:45 +0900 Subject: [PATCH 18/34] aot: Disable musttail for mips (#2457) Fixes https://github.com/bytecodealliance/wasm-micro-runtime/issues/2412 --- core/iwasm/compilation/aot_llvm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index b47c198c..fb9fc0be 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -146,6 +146,13 @@ aot_target_precheck_can_use_musttail(const AOTCompContext *comp_ctx) */ return false; } + if (!strcmp(comp_ctx->target_arch, "mips")) { + /* + * cf. + * https://github.com/bytecodealliance/wasm-micro-runtime/issues/2412 + */ + return false; + } /* * x86-64/i386: true * From c820643b2b94825833589b6fc6686a3ae3241318 Mon Sep 17 00:00:00 2001 From: TianlongLiang <111852609+TianlongLiang@users.noreply.github.com> Date: Wed, 16 Aug 2023 05:28:45 +0200 Subject: [PATCH 19/34] wamrc: More friendly to print help info (#2451) Allow wamrc to print help info like below: ```bash wamrc --target=help wamrc --target-abi=help wamrc --target= --cpu=help wamrc --target= --cpu=help --cpu-features=+help ``` --- core/iwasm/compilation/aot_llvm.c | 10 ++++++ wamr-compiler/main.c | 53 ++++++++++++++++++++++++------- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index fb9fc0be..f4bc8500 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -2752,6 +2752,16 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) aot_set_last_error("create LLVM target machine failed."); goto fail; } + + /* If only to create target machine for querying information, early stop + */ + if ((arch && !strcmp(arch, "help")) || (abi && !strcmp(abi, "help")) + || (cpu && !strcmp(cpu, "help")) + || (features && !strcmp(features, "+help"))) { + LOG_DEBUG( + "create LLVM target machine only for printing help info."); + goto fail; + } } triple = LLVMGetTargetMachineTriple(comp_ctx->target_machine); diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index 8d2c9357..0d85b99c 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -193,6 +193,8 @@ print_help() printf("Examples: wamrc -o test.aot test.wasm\n"); printf(" wamrc --target=i386 -o test.aot test.wasm\n"); printf(" wamrc --target=i386 --format=object -o test.o test.wasm\n"); + printf(" wamrc --target-abi=help\n"); + printf(" wamrc --target=x86_64 --cpu=help\n"); } /* clang-format on */ @@ -296,6 +298,12 @@ resolve_segue_flags(char *str_flags) return segue_flags; } +/* When print help info for target/cpu/target-abi/cpu-features, load this dummy + * wasm file content rather than from an input file, the dummy wasm file content + * is: magic header + version number */ +static unsigned char dummy_wasm_file[8] = { 0x00, 0x61, 0x73, 0x6D, + 0x01, 0x00, 0x00, 0x00 }; + int main(int argc, char *argv[]) { @@ -309,7 +317,7 @@ main(int argc, char *argv[]) AOTCompOption option = { 0 }; char error_buf[128]; int log_verbose_level = 2; - bool sgx_mode = false, size_level_set = false; + bool sgx_mode = false, size_level_set = false, use_dummy_wasm = false; int exit_status = EXIT_FAILURE; #if BH_HAS_DLFCN const char *native_lib_list[8] = { NULL }; @@ -342,21 +350,33 @@ main(int argc, char *argv[]) if (argv[0][9] == '\0') PRINT_HELP_AND_EXIT(); option.target_arch = argv[0] + 9; + if (!strcmp(option.target_arch, "help")) { + use_dummy_wasm = true; + } } else if (!strncmp(argv[0], "--target-abi=", 13)) { if (argv[0][13] == '\0') PRINT_HELP_AND_EXIT(); option.target_abi = argv[0] + 13; + if (!strcmp(option.target_abi, "help")) { + use_dummy_wasm = true; + } } else if (!strncmp(argv[0], "--cpu=", 6)) { if (argv[0][6] == '\0') PRINT_HELP_AND_EXIT(); option.target_cpu = argv[0] + 6; + if (!strcmp(option.target_cpu, "help")) { + use_dummy_wasm = true; + } } else if (!strncmp(argv[0], "--cpu-features=", 15)) { if (argv[0][15] == '\0') PRINT_HELP_AND_EXIT(); option.cpu_features = argv[0] + 15; + if (!strcmp(option.cpu_features, "+help")) { + use_dummy_wasm = true; + } } else if (!strncmp(argv[0], "--opt-level=", 12)) { if (argv[0][12] == '\0') @@ -516,7 +536,7 @@ main(int argc, char *argv[]) PRINT_HELP_AND_EXIT(); } - if (argc == 0 || !out_file_name) + if (!use_dummy_wasm && (argc == 0 || !out_file_name)) PRINT_HELP_AND_EXIT(); if (!size_level_set) { @@ -544,11 +564,13 @@ main(int argc, char *argv[]) option.is_sgx_platform = true; } - wasm_file_name = argv[0]; + if (!use_dummy_wasm) { + wasm_file_name = argv[0]; - if (!strcmp(wasm_file_name, out_file_name)) { - printf("Error: input file and output file are the same"); - return -1; + if (!strcmp(wasm_file_name, out_file_name)) { + printf("Error: input file and output file are the same"); + return -1; + } } memset(&init_args, 0, sizeof(RuntimeInitArgs)); @@ -574,10 +596,17 @@ main(int argc, char *argv[]) bh_print_time("Begin to load wasm file"); - /* load WASM byte buffer from WASM bin file */ - if (!(wasm_file = - (uint8 *)bh_read_file_to_buffer(wasm_file_name, &wasm_file_size))) - goto fail1; + if (use_dummy_wasm) { + /* load WASM byte buffer from dummy buffer */ + wasm_file_size = sizeof(dummy_wasm_file); + wasm_file = dummy_wasm_file; + } + else { + /* load WASM byte buffer from WASM bin file */ + if (!(wasm_file = (uint8 *)bh_read_file_to_buffer(wasm_file_name, + &wasm_file_size))) + goto fail1; + } if (get_package_type(wasm_file, wasm_file_size) != Wasm_Module_Bytecode) { printf("Invalid file type: expected wasm file but got other\n"); @@ -659,7 +688,9 @@ fail3: fail2: /* free the file buffer */ - wasm_runtime_free(wasm_file); + if (!use_dummy_wasm) { + wasm_runtime_free(wasm_file); + } fail1: #if BH_HAS_DLFCN From 4ce675aacd7d69664c74f9daaf7badab38c4c80f Mon Sep 17 00:00:00 2001 From: Maks Litskevich Date: Thu, 17 Aug 2023 03:31:05 +0100 Subject: [PATCH 20/34] Add another wamr test (#2411) Follows up #2364 where we discussed that we might want to have a test which has really short thread function and creates many threads. --- .../compilation_on_android_ubuntu.yml | 4 +- .github/workflows/nightly_run.yml | 4 +- .../libraries/lib-wasi-threads/test/build.sh | 3 +- .../libraries/lib-wasi-threads/test/skip.json | 3 +- .../lib-wasi-threads/test/spawn_stress_test.c | 9 +- .../test/stress_test_threads_creation.c | 93 +++++++++++++++++++ .../wasi-test-script/run_wasi_tests.sh | 33 ++++--- 7 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 core/iwasm/libraries/lib-wasi-threads/test/stress_test_threads_creation.c diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index ee22af86..bbb2c4b3 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -537,7 +537,7 @@ jobs: working-directory: ./core/iwasm/libraries/lib-socket/test/ - name: run tests - timeout-minutes: 20 + timeout-minutes: 30 run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites @@ -553,7 +553,7 @@ jobs: sudo apt install -y g++-multilib lib32gcc-9-dev - name: run tests x86_32 - timeout-minutes: 20 + timeout-minutes: 30 if: env.TEST_ON_X86_32 == 'true' run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index b6cc15dc..0b82bb58 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -605,7 +605,7 @@ jobs: working-directory: ./core/iwasm/libraries/lib-socket/test/ - name: run tests - timeout-minutes: 20 + timeout-minutes: 40 run: ./test_wamr.sh ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites @@ -621,7 +621,7 @@ jobs: sudo apt install -y g++-multilib lib32gcc-9-dev - name: run tests x86_32 - timeout-minutes: 20 + timeout-minutes: 40 if: env.TEST_ON_X86_32 == 'true' run: ./test_wamr.sh ${{ env.X86_32_TARGET_TEST_OPTIONS }} ${{ matrix.test_option }} -t ${{ matrix.running_mode }} working-directory: ./tests/wamr-test-suites \ No newline at end of file diff --git a/core/iwasm/libraries/lib-wasi-threads/test/build.sh b/core/iwasm/libraries/lib-wasi-threads/test/build.sh index 7c9a773c..78a08105 100755 --- a/core/iwasm/libraries/lib-wasi-threads/test/build.sh +++ b/core/iwasm/libraries/lib-wasi-threads/test/build.sh @@ -34,7 +34,7 @@ while [[ $# -gt 0 ]]; do done # Stress tests names -thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm") +thread_start_file_exclusions=("spawn_stress_test.wasm" "linear_memory_size_update.wasm" "stress_test_threads_creation.wasm") for test_c in *.c; do test_wasm="$(basename $test_c .c).wasm" @@ -56,6 +56,7 @@ for test_c in *.c; do echo "Compiling $test_c to $test_wasm" $CC \ -target wasm32-wasi-threads \ + -O2 \ -pthread -ftls-model=local-exec \ -z stack-size=32768 \ -Wl,--export=__heap_base \ diff --git a/core/iwasm/libraries/lib-wasi-threads/test/skip.json b/core/iwasm/libraries/lib-wasi-threads/test/skip.json index 87db7667..86a3ba25 100644 --- a/core/iwasm/libraries/lib-wasi-threads/test/skip.json +++ b/core/iwasm/libraries/lib-wasi-threads/test/skip.json @@ -1,5 +1,6 @@ { "lib-wasi-threads tests": { - "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently" + "spawn_stress_test": "Stress tests are incompatible with the other part and executed differently", + "stress_test_threads_creation": "Stress tests are incompatible with the other part and executed differently" } } diff --git a/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c b/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c index 35175fc3..fa49ac48 100644 --- a/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c +++ b/core/iwasm/libraries/lib-wasi-threads/test/spawn_stress_test.c @@ -18,8 +18,9 @@ enum CONSTANTS { NUM_ITER = 100000, - NUM_RETRY = 5, + NUM_RETRY = 8, MAX_NUM_THREADS = 8, + RETRY_SLEEP_TIME_US = 2000, }; unsigned prime_numbers_count = 0; @@ -62,11 +63,13 @@ void spawn_thread(pthread_t *thread, unsigned int *arg) { int status_code = -1; + int timeout_us = RETRY_SLEEP_TIME_US; for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) { status_code = pthread_create(thread, NULL, &check_if_prime, arg); assert(status_code == 0 || status_code == EAGAIN); if (status_code == EAGAIN) { - usleep(2000); + usleep(timeout_us); + timeout_us *= 2; } } @@ -95,7 +98,7 @@ main(int argc, char **argv) args[thread_num] = factorised_number; - usleep(2000); + usleep(RETRY_SLEEP_TIME_US); spawn_thread(&threads[thread_num], &args[thread_num]); assert(threads[thread_num] != 0); } diff --git a/core/iwasm/libraries/lib-wasi-threads/test/stress_test_threads_creation.c b/core/iwasm/libraries/lib-wasi-threads/test/stress_test_threads_creation.c new file mode 100644 index 00000000..ef148f13 --- /dev/null +++ b/core/iwasm/libraries/lib-wasi-threads/test/stress_test_threads_creation.c @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + */ + +#include +#include +#include +#include +#include + +enum CONSTANTS { + NUM_ITER = 200000, + NUM_RETRY = 8, + MAX_NUM_THREADS = 8, + RETRY_SLEEP_TIME_US = 4000, + SECOND = 1000 * 1000 * 1000 +}; + +int threads_executed = 0; +unsigned int threads_creation_tried = 0; +unsigned int threads_in_use = 0; + +void * +thread_func(void *arg) +{ + (void)(arg); + __atomic_fetch_add(&threads_executed, 1, __ATOMIC_RELAXED); + __atomic_fetch_sub(&threads_in_use, 1, __ATOMIC_SEQ_CST); + return NULL; +} + +void +spawn_thread(pthread_t *thread) +{ + int status_code = -1; + int timeout_us = RETRY_SLEEP_TIME_US; + for (int tries = 0; status_code != 0 && tries < NUM_RETRY; ++tries) { + status_code = pthread_create(thread, NULL, &thread_func, NULL); + __atomic_fetch_add(&threads_creation_tried, 1, __ATOMIC_RELAXED); + + assert(status_code == 0 || status_code == EAGAIN); + if (status_code == EAGAIN) { + usleep(timeout_us); + timeout_us *= 2; + } + } + + assert(status_code == 0 && "Thread creation should succeed"); +} + +int +main(int argc, char **argv) +{ + double percentage = 0.1; + + for (int iter = 0; iter < NUM_ITER; ++iter) { + if (iter > NUM_ITER * percentage) { + fprintf(stderr, "Spawning stress test is %d%% finished\n", + (unsigned int)(percentage * 100)); + percentage += 0.1; + } + while (__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) + == MAX_NUM_THREADS) { + usleep(100); + } + + __atomic_fetch_add(&threads_in_use, 1, __ATOMIC_SEQ_CST); + pthread_t tmp; + spawn_thread(&tmp); + pthread_detach(tmp); + } + + while ((__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) != 0)) { + __builtin_wasm_memory_atomic_wait32(&threads_in_use, 0, SECOND); + } + + assert(__atomic_load_n(&threads_in_use, __ATOMIC_SEQ_CST) == 0); + + // Validation + assert(threads_creation_tried >= threads_executed + && "Test executed more threads than were created"); + assert((1. * threads_creation_tried) / threads_executed < 2.5 + && "Ensuring that we're retrying thread creation less than 2.5 " + "times on average "); + + fprintf(stderr, + "Spawning stress test finished successfully executed %d threads " + "with retry ratio %f\n", + threads_creation_tried, + (1. * threads_creation_tried) / threads_executed); + return 0; +} diff --git a/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh b/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh index 9f13ee08..454fbb14 100755 --- a/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh +++ b/tests/wamr-test-suites/wasi-test-script/run_wasi_tests.sh @@ -14,6 +14,7 @@ readonly WAMR_DIR="${WORK_DIR}/../../../.." readonly IWASM_CMD="${WORK_DIR}/../../../../product-mini/platforms/${PLATFORM}/build/iwasm \ --allow-resolve=google-public-dns-a.google.com \ --addr-pool=::1/128,127.0.0.1/32" + readonly IWASM_CMD_STRESS="${IWASM_CMD} --max-threads=8" readonly WAMRC_CMD="${WORK_DIR}/../../../../wamr-compiler/build/wamrc" readonly C_TESTS="tests/c/testsuite/" @@ -21,18 +22,22 @@ readonly ASSEMBLYSCRIPT_TESTS="tests/assemblyscript/testsuite/" readonly THREAD_PROPOSAL_TESTS="tests/proposals/wasi-threads/" readonly THREAD_INTERNAL_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-wasi-threads/test/" readonly LIB_SOCKET_TESTS="${WAMR_DIR}/core/iwasm/libraries/lib-socket/test/" +readonly STRESS_TESTS=("spawn_stress_test.wasm" "stress_test_threads_creation.wasm") run_aot_tests () { local tests=("$@") + local iwasm="${IWASM_CMD}" for test_wasm in ${tests[@]}; do local extra_stress_flags="" - if [[ "$test_wasm" =~ "stress" ]]; then - extra_stress_flags="--max-threads=8" - fi + for stress_test in "${STRESS_TESTS[@]}"; do + if [ "$test_wasm" == "$stress_test" ]; then + iwasm="${IWASM_CMD_STRESS}" + fi + done test_aot="${test_wasm%.wasm}.aot" test_json="${test_wasm%.wasm}.json" - + if [ -f ${test_wasm} ]; then expected=$(jq .exit_code ${test_json}) fi @@ -48,7 +53,6 @@ run_aot_tests () { fi ${IWASM_CMD} $extra_stress_flags $test_aot - ret=${PIPESTATUS[0]} echo "expected=$expected, actual=$ret" @@ -62,15 +66,18 @@ if [[ $MODE != "aot" ]];then python3 -m venv wasi-env && source wasi-env/bin/activate python3 -m pip install -r test-runner/requirements.txt - # Stress test requires max-threads=8 so it's run separately - if [[ -e "${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm" ]]; then - ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}spawn_stress_test.wasm - ret=${PIPESTATUS[0]} - if [ "${ret}" -ne 0 ]; then - echo "Stress test spawn_stress_test FAILED with code " ${ret} - exit_code=${ret} + # Stress tests require max-threads=8 so they're executed separately + for stress_test in "${STRESS_TESTS[@]}"; do + if [[ -e "${THREAD_INTERNAL_TESTS}${stress_test}" ]]; then + echo "${stress_test}" is a stress test + ${IWASM_CMD_STRESS} ${THREAD_INTERNAL_TESTS}${stress_test} + ret=${PIPESTATUS[0]} + if [ "${ret}" -ne 0 ]; then + echo "Stress test ${stress_test} FAILED with code " ${ret} + exit_code=${ret} + fi fi - fi + done TEST_RUNTIME_EXE="${IWASM_CMD}" python3 test-runner/wasi_test_runner.py \ -r adapters/wasm-micro-runtime.py \ From e2f8721ec97d25f4f64a92563f42c69c8fc51b4b Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Thu, 17 Aug 2023 10:54:02 +0800 Subject: [PATCH 21/34] Fix issues reported by Coverity and clear windows warnings (#2467) --- core/iwasm/compilation/aot_emit_aot_file.c | 15 +++++++++------ core/iwasm/interpreter/wasm_runtime.c | 5 +++-- .../libraries/lib-wasi-threads/tid_allocator.c | 11 +++++++---- core/shared/platform/common/posix/posix_socket.c | 2 +- product-mini/platforms/posix/main.c | 2 +- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index f1a5c479..4a5006ad 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -2605,9 +2605,16 @@ aot_resolve_stack_sizes(AOTCompContext *comp_ctx, AOTObjectData *obj_data) while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) { if ((name = LLVMGetSymbolName(sym_itr)) - && !strcmp(name, aot_stack_sizes_alias_name)) { + && (!strcmp(name, aot_stack_sizes_alias_name) + /* symbol of COFF32 starts with "_" */ + || (obj_data->target_info.bin_type == AOT_COFF32_BIN_TYPE + && !strncmp(name, "_", 1) + && !strcmp(name + 1, aot_stack_sizes_alias_name)))) { uint64 sz = LLVMGetSymbolSize(sym_itr); - if (sz != sizeof(uint32) * obj_data->func_count) { + if (sz != sizeof(uint32) * obj_data->func_count + /* sz of COFF64/COFF32 is 0, ignore the check */ + && obj_data->target_info.bin_type != AOT_COFF64_BIN_TYPE + && obj_data->target_info.bin_type != AOT_COFF32_BIN_TYPE) { aot_set_last_error("stack_sizes had unexpected size."); goto fail; } @@ -2642,10 +2649,6 @@ aot_resolve_stack_sizes(AOTCompContext *comp_ctx, AOTObjectData *obj_data) goto fail; } } - if (addr > UINT32_MAX) { - aot_set_last_error("too large stack_sizes offset."); - goto fail; - } /* * Record section/offset and construct a copy of stack_sizes. * aot_emit_object_data_section_info will emit this copy. diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 2b88c398..b71b608a 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -1913,8 +1913,9 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } if (memory_data) { - bh_memcpy_s(memory_data + base_offset, memory_size - base_offset, - data_seg->data, length); + bh_memcpy_s(memory_data + base_offset, + (uint32)memory_size - base_offset, data_seg->data, + length); } } diff --git a/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c b/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c index 4d53da0c..dc2d4f1b 100644 --- a/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c +++ b/core/iwasm/libraries/lib-wasi-threads/tid_allocator.c @@ -21,7 +21,8 @@ tid_allocator_init(TidAllocator *tid_allocator) return false; for (int64 i = tid_allocator->pos - 1; i >= 0; i--) - tid_allocator->ids[i] = TID_MIN + (tid_allocator->pos - 1 - i); + tid_allocator->ids[i] = + (uint32)(TID_MIN + (tid_allocator->pos - 1 - i)); return true; } @@ -54,7 +55,8 @@ tid_allocator_get_tid(TidAllocator *tid_allocator) LOG_ERROR("Overflow detected during realloc"); return -1; } - int32 *tmp = wasm_runtime_realloc(tid_allocator->ids, realloc_size); + int32 *tmp = + wasm_runtime_realloc(tid_allocator->ids, (uint32)realloc_size); if (tmp == NULL) { LOG_ERROR("Thread ID allocator realloc failed"); return -1; @@ -64,7 +66,8 @@ tid_allocator_get_tid(TidAllocator *tid_allocator) tid_allocator->pos = new_size - old_size; tid_allocator->ids = tmp; for (int64 i = tid_allocator->pos - 1; i >= 0; i--) - tid_allocator->ids[i] = TID_MIN + (tid_allocator->size - 1 - i); + tid_allocator->ids[i] = + (uint32)(TID_MIN + (tid_allocator->size - 1 - i)); } // Pop available thread identifier from the stack @@ -77,4 +80,4 @@ tid_allocator_release_tid(TidAllocator *tid_allocator, int32 thread_id) // Release thread identifier by pushing it into the stack bh_assert(tid_allocator->pos < tid_allocator->size); tid_allocator->ids[tid_allocator->pos++] = thread_id; -} \ No newline at end of file +} diff --git a/core/shared/platform/common/posix/posix_socket.c b/core/shared/platform/common/posix/posix_socket.c index c018588a..d0a232bc 100644 --- a/core/shared/platform/common/posix/posix_socket.c +++ b/core/shared/platform/common/posix/posix_socket.c @@ -275,7 +275,7 @@ os_socket_recv_from(bh_socket_t socket, void *buf, unsigned int len, int flags, return -1; } } - else { + else if (src_addr) { memset(src_addr, 0, sizeof(*src_addr)); } diff --git a/product-mini/platforms/posix/main.c b/product-mini/platforms/posix/main.c index b79af9b8..5d20fd29 100644 --- a/product-mini/platforms/posix/main.c +++ b/product-mini/platforms/posix/main.c @@ -330,7 +330,7 @@ load_native_lib(const char *name) if (ret != 0) { LOG_WARNING("warning: `init_native_lib` function from native " "lib %s failed with %d", - name); + name, ret); goto fail; } } From 7baaed2fb82104e98c926965631618fecda9062e Mon Sep 17 00:00:00 2001 From: Xu Jun Date: Thu, 17 Aug 2023 19:49:35 +0800 Subject: [PATCH 22/34] Fix opcode overwrite issue in fast interp (#2476) --- core/iwasm/interpreter/wasm_loader.c | 5 +++-- core/iwasm/interpreter/wasm_mini_loader.c | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/iwasm/interpreter/wasm_loader.c b/core/iwasm/interpreter/wasm_loader.c index 08441ec3..c775ee6c 100644 --- a/core/iwasm/interpreter/wasm_loader.c +++ b/core/iwasm/interpreter/wasm_loader.c @@ -8064,6 +8064,9 @@ re_scan: case WASM_OP_SELECT_T: { uint8 vec_len, ref_type; +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif read_leb_uint32(p, p_end, vec_len); if (vec_len != 1) { @@ -8086,8 +8089,6 @@ re_scan: #if WASM_ENABLE_FAST_INTERP != 0 if (loader_ctx->p_code_compiled) { uint8 opcode_tmp = WASM_OP_SELECT; - uint8 *p_code_compiled_tmp = - loader_ctx->p_code_compiled - 2; if (ref_type == VALUE_TYPE_V128) { #if (WASM_ENABLE_SIMD == 0) \ diff --git a/core/iwasm/interpreter/wasm_mini_loader.c b/core/iwasm/interpreter/wasm_mini_loader.c index 0aba1462..9f568e13 100644 --- a/core/iwasm/interpreter/wasm_mini_loader.c +++ b/core/iwasm/interpreter/wasm_mini_loader.c @@ -6233,6 +6233,9 @@ re_scan: case WASM_OP_SELECT_T: { uint8 vec_len, ref_type; +#if WASM_ENABLE_FAST_INTERP != 0 + uint8 *p_code_compiled_tmp = loader_ctx->p_code_compiled; +#endif read_leb_uint32(p, p_end, vec_len); if (vec_len != 1) { @@ -6255,8 +6258,6 @@ re_scan: #if WASM_ENABLE_FAST_INTERP != 0 if (loader_ctx->p_code_compiled) { uint8 opcode_tmp = WASM_OP_SELECT; - uint8 *p_code_compiled_tmp = - loader_ctx->p_code_compiled - 2; if (ref_type == VALUE_TYPE_F64 || ref_type == VALUE_TYPE_I64) From 6716c23dd17aa02e2e500740ab11bddf5b49ffc5 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 18 Aug 2023 09:56:00 +0800 Subject: [PATCH 23/34] wamrc: Fix windows relocation to `aot_func_internal#n` (#2474) AOT relocation to aot_func_internal#n is generated by wamrc --bounds-checks=1. Resolve the issue by applying the relocation in the compilation stage by wamrc and don't generate these relocations in the AOT file. Fixes #2471. --- core/iwasm/aot/arch/aot_reloc_x86_32.c | 3 + core/iwasm/aot/arch/aot_reloc_x86_64.c | 2 +- core/iwasm/compilation/aot_emit_aot_file.c | 144 ++++++++++++++++++++- 3 files changed, 142 insertions(+), 7 deletions(-) diff --git a/core/iwasm/aot/arch/aot_reloc_x86_32.c b/core/iwasm/aot/arch/aot_reloc_x86_32.c index 9246836c..0a423c39 100644 --- a/core/iwasm/aot/arch/aot_reloc_x86_32.c +++ b/core/iwasm/aot/arch/aot_reloc_x86_32.c @@ -164,6 +164,9 @@ apply_relocation(AOTModule *module, uint8 *target_section_addr, (uint32)((uintptr_t)symbol_addr + (intptr_t)reloc_addend - (uintptr_t)(target_section_addr + (uint32)reloc_offset) +#if defined(BH_PLATFORM_WINDOWS) + - sizeof(int32) +#endif + value); /* S + A - P */ break; } diff --git a/core/iwasm/aot/arch/aot_reloc_x86_64.c b/core/iwasm/aot/arch/aot_reloc_x86_64.c index 1221a629..1bf5eb59 100644 --- a/core/iwasm/aot/arch/aot_reloc_x86_64.c +++ b/core/iwasm/aot/arch/aot_reloc_x86_64.c @@ -242,7 +242,7 @@ apply_relocation(AOTModule *module, uint8 *target_section_addr, - (uintptr_t)(target_section_addr + reloc_offset)); } else { - target_addr = (intptr_t) /* L + A - P */ + target_addr = (intptr_t) /* S + A - P */ ((uintptr_t)symbol_addr + reloc_addend - (uintptr_t)(target_section_addr + reloc_offset)); } diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 4a5006ad..1b1e4403 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -93,7 +93,10 @@ check_utf8_str(const uint8 *str, uint32 len) /* Internal function in object file */ typedef struct AOTObjectFunc { char *func_name; + /* text offset of aot_func#n */ uint64 text_offset; + /* text offset of aot_func_internal#n */ + uint64 text_offset_of_aot_func_internal; } AOTObjectFunc; /* Symbol table list node */ @@ -637,13 +640,33 @@ get_relocation_size(AOTRelocation *relocation, bool is_32bin) } static uint32 -get_relocations_size(AOTRelocation *relocations, uint32 relocation_count, +get_relocations_size(AOTObjectData *obj_data, + AOTRelocationGroup *relocation_group, + AOTRelocation *relocations, uint32 relocation_count, bool is_32bin) { AOTRelocation *relocation = relocations; uint32 size = 0, i; for (i = 0; i < relocation_count; i++, relocation++) { + /* ignore the relocations to aot_func_internal#n in text section + for windows platform since they will be applied in + aot_emit_text_section */ + if (!strcmp(relocation_group->section_name, ".text") + && !strncmp(relocation->symbol_name, AOT_FUNC_INTERNAL_PREFIX, + strlen(AOT_FUNC_INTERNAL_PREFIX)) + && ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6) + /* Windows AOT_COFF64_BIN_TYPE */ + && obj_data->target_info.bin_type == 6 + /* IMAGE_REL_AMD64_REL32 in windows x86_64 */ + && relocation->relocation_type == 4) + || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4) + /* Windows AOT_COFF32_BIN_TYPE */ + && obj_data->target_info.bin_type == 4 + /* IMAGE_REL_I386_REL32 in windows x86_32 */ + && relocation->relocation_type == 20))) { + continue; + } size = align_uint(size, 4); size += get_relocation_size(relocation, is_32bin); } @@ -651,19 +674,22 @@ get_relocations_size(AOTRelocation *relocations, uint32 relocation_count, } static uint32 -get_relocation_group_size(AOTRelocationGroup *relocation_group, bool is_32bin) +get_relocation_group_size(AOTObjectData *obj_data, + AOTRelocationGroup *relocation_group, bool is_32bin) { uint32 size = 0; /* section name index + relocation count + relocations */ size += (uint32)sizeof(uint32); size += (uint32)sizeof(uint32); - size += get_relocations_size(relocation_group->relocations, + size += get_relocations_size(obj_data, relocation_group, + relocation_group->relocations, relocation_group->relocation_count, is_32bin); return size; } static uint32 -get_relocation_groups_size(AOTRelocationGroup *relocation_groups, +get_relocation_groups_size(AOTObjectData *obj_data, + AOTRelocationGroup *relocation_groups, uint32 relocation_group_count, bool is_32bin) { AOTRelocationGroup *relocation_group = relocation_groups; @@ -671,7 +697,7 @@ get_relocation_groups_size(AOTRelocationGroup *relocation_groups, for (i = 0; i < relocation_group_count; i++, relocation_group++) { size = align_uint(size, 4); - size += get_relocation_group_size(relocation_group, is_32bin); + size += get_relocation_group_size(obj_data, relocation_group, is_32bin); } return size; } @@ -864,7 +890,7 @@ get_relocation_section_size(AOTCompContext *comp_ctx, AOTObjectData *obj_data) /* relocation group count + symbol_table + relocation groups */ return (uint32)sizeof(uint32) + symbol_table_size - + get_relocation_groups_size(relocation_groups, + + get_relocation_groups_size(obj_data, relocation_groups, relocation_group_count, is_32bit_binary(obj_data)); } @@ -1734,6 +1760,10 @@ aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, uint32 section_size = get_text_section_size(obj_data); uint32 offset = *p_offset; uint8 placeholder = 0; + AOTRelocationGroup *relocation_group; + AOTRelocation *relocation; + uint32 i, j, relocation_count; + uint8 *text; *p_offset = offset = align_uint(offset, 4); @@ -1747,6 +1777,8 @@ aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, EMIT_BUF(&placeholder, 1); } + text = buf + offset; + if (obj_data->text_size > 0) { EMIT_BUF(obj_data->text, obj_data->text_size); while (offset & 3) @@ -1768,6 +1800,67 @@ aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, return false; } + /* apply relocations to aot_func_internal#n in text section for + windows platform */ + if ((!strncmp(obj_data->comp_ctx->target_arch, "x86_64", 6) + /* Windows AOT_COFF64_BIN_TYPE */ + && obj_data->target_info.bin_type == 6) + || (!strncmp(obj_data->comp_ctx->target_arch, "i386", 4) + /* Windows AOT_COFF32_BIN_TYPE */ + && obj_data->target_info.bin_type == 4)) { + relocation_group = obj_data->relocation_groups; + for (i = 0; i < obj_data->relocation_group_count; + i++, relocation_group++) { + /* relocation in text section */ + if (!strcmp(relocation_group->section_name, ".text")) { + relocation = relocation_group->relocations; + relocation_count = relocation_group->relocation_count; + for (j = 0; j < relocation_count; j++) { + /* relocation to aot_func_internal#n */ + if (str_starts_with(relocation->symbol_name, + AOT_FUNC_INTERNAL_PREFIX) + && ((obj_data->target_info.bin_type + == 6 /* AOT_COFF64_BIN_TYPE */ + && relocation->relocation_type + == 4 /* IMAGE_REL_AMD64_REL32 */) + || (obj_data->target_info.bin_type + == 4 /* AOT_COFF32_BIN_TYPE */ + && obj_data->target_info.bin_type + == 20 /* IMAGE_REL_I386_REL32 */))) { + uint32 func_idx = + atoi(relocation->symbol_name + + strlen(AOT_FUNC_INTERNAL_PREFIX)); + uint64 text_offset, reloc_offset, reloc_addend; + + bh_assert(func_idx < obj_data->func_count); + + text_offset = obj_data->funcs[func_idx] + .text_offset_of_aot_func_internal; + reloc_offset = relocation->relocation_offset; + reloc_addend = relocation->relocation_addend; + /* S + A - P */ + *(uint32 *)(text + reloc_offset) = + (uint32)(text_offset + reloc_addend - reloc_offset + - 4); + + /* remove current relocation as it has been applied */ + if (j < relocation_count - 1) { + uint32 move_size = + (uint32)(sizeof(AOTRelocation) + * (relocation_count - 1 - j)); + bh_memmove_s(relocation, move_size, relocation + 1, + move_size); + } + relocation_group->relocation_count--; + } + else { + relocation++; + } + } + } + } + } + *p_offset = offset; return true; @@ -2773,6 +2866,7 @@ aot_resolve_functions(AOTCompContext *comp_ctx, AOTObjectData *obj_data) while (!LLVMObjectFileIsSymbolIteratorAtEnd(obj_data->binary, sym_itr)) { if ((name = (char *)LLVMGetSymbolName(sym_itr)) && str_starts_with(name, prefix)) { + /* symbol aot_func#n */ func_index = (uint32)atoi(name + strlen(prefix)); if (func_index < obj_data->func_count) { LLVMSectionIteratorRef contain_section; @@ -2807,6 +2901,44 @@ aot_resolve_functions(AOTCompContext *comp_ctx, AOTObjectData *obj_data) } } } + else if ((name = (char *)LLVMGetSymbolName(sym_itr)) + && str_starts_with(name, AOT_FUNC_INTERNAL_PREFIX)) { + /* symbol aot_func_internal#n */ + func_index = (uint32)atoi(name + strlen(AOT_FUNC_INTERNAL_PREFIX)); + if (func_index < obj_data->func_count) { + LLVMSectionIteratorRef contain_section; + char *contain_section_name; + + func = obj_data->funcs + func_index; + + if (!(contain_section = LLVMObjectFileCopySectionIterator( + obj_data->binary))) { + aot_set_last_error("llvm get section iterator failed."); + LLVMDisposeSymbolIterator(sym_itr); + return false; + } + LLVMMoveToContainingSection(contain_section, sym_itr); + contain_section_name = + (char *)LLVMGetSectionName(contain_section); + LLVMDisposeSectionIterator(contain_section); + + if (!strcmp(contain_section_name, ".text.unlikely.")) { + func->text_offset_of_aot_func_internal = + align_uint(obj_data->text_size, 4) + + LLVMGetSymbolAddress(sym_itr); + } + else if (!strcmp(contain_section_name, ".text.hot.")) { + func->text_offset_of_aot_func_internal = + align_uint(obj_data->text_size, 4) + + align_uint(obj_data->text_unlikely_size, 4) + + LLVMGetSymbolAddress(sym_itr); + } + else { + func->text_offset_of_aot_func_internal = + LLVMGetSymbolAddress(sym_itr); + } + } + } LLVMMoveToNextSymbol(sym_itr); } LLVMDisposeSymbolIterator(sym_itr); From 2cb701f7f3b4f8b2d0f65b499677338034b2403b Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 18 Aug 2023 10:16:03 +0800 Subject: [PATCH 24/34] Fix windows AOT hw bound check (#2475) When AOT out of bound linear memory access or stack overflow occurs, the call stack of AOT functions cannot be unwound currently, so from the exception handler, runtime cannot jump back into the place that calls the AOT function. We temporarily skip the current instruction and let AOT code continue to run and return to caller as soon as possible. And use the zydis library the decode the current instruction to get its size. And remove using RtlAddFunctionTable to register the AOT functions since it doesn't work currently. --- core/iwasm/aot/aot_loader.c | 53 --------- core/iwasm/aot/aot_runtime.h | 32 ------ core/iwasm/aot/arch/aot_reloc_x86_64.c | 15 --- core/iwasm/aot/iwasm_aot.cmake | 46 +++++++- core/iwasm/common/wasm_runtime_common.c | 125 ++++++++++++++++----- core/iwasm/compilation/aot_emit_function.c | 22 +++- 6 files changed, 159 insertions(+), 134 deletions(-) diff --git a/core/iwasm/aot/aot_loader.c b/core/iwasm/aot/aot_loader.c index 50e36f9e..0ce2cf9d 100644 --- a/core/iwasm/aot/aot_loader.c +++ b/core/iwasm/aot/aot_loader.c @@ -1644,27 +1644,6 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, const uint8 *p = buf, *p_end = buf_end; uint32 i; uint64 size, text_offset; -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - RUNTIME_FUNCTION *rtl_func_table; - AOTUnwindInfo *unwind_info; - uint32 unwind_info_offset = module->code_size - sizeof(AOTUnwindInfo); - uint32 unwind_code_offset = unwind_info_offset - PLT_ITEM_SIZE; -#endif - -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - unwind_info = (AOTUnwindInfo *)((uint8 *)module->code + module->code_size - - sizeof(AOTUnwindInfo)); - unwind_info->Version = 1; - unwind_info->Flags = UNW_FLAG_NHANDLER; - *(uint32 *)&unwind_info->UnwindCode[0] = unwind_code_offset; - - size = sizeof(RUNTIME_FUNCTION) * (uint64)module->func_count; - if (size > 0 - && !(rtl_func_table = module->rtl_func_table = - loader_malloc(size, error_buf, error_buf_size))) { - return false; - } -#endif size = sizeof(void *) * (uint64)module->func_count; if (size > 0 @@ -1691,33 +1670,9 @@ load_function_section(const uint8 *buf, const uint8 *buf_end, AOTModule *module, #if defined(BUILD_TARGET_THUMB) || defined(BUILD_TARGET_THUMB_VFP) /* bits[0] of thumb function address must be 1 */ module->func_ptrs[i] = (void *)((uintptr_t)module->func_ptrs[i] | 1); -#endif -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - rtl_func_table[i].BeginAddress = (DWORD)text_offset; - if (i > 0) { - rtl_func_table[i - 1].EndAddress = rtl_func_table[i].BeginAddress; - } - rtl_func_table[i].UnwindInfoAddress = (DWORD)unwind_info_offset; #endif } -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - if (module->func_count > 0) { - uint32 plt_table_size = - module->is_indirect_mode ? 0 : get_plt_table_size(); - rtl_func_table[module->func_count - 1].EndAddress = - (DWORD)(module->code_size - plt_table_size); - - if (!RtlAddFunctionTable(rtl_func_table, module->func_count, - (DWORD64)(uintptr_t)module->code)) { - set_error_buf(error_buf, error_buf_size, - "add dynamic function table failed"); - return false; - } - module->rtl_func_table_registered = true; - } -#endif - /* Set start function when function pointers are resolved */ if (module->start_func_index != (uint32)-1) { if (module->start_func_index >= module->import_func_count) @@ -3261,14 +3216,6 @@ aot_unload(AOTModule *module) } #endif -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - if (module->rtl_func_table) { - if (module->rtl_func_table_registered) - RtlDeleteFunctionTable(module->rtl_func_table); - wasm_runtime_free(module->rtl_func_table); - } -#endif - #if (defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)) \ && !defined(BH_PLATFORM_WINDOWS) { diff --git a/core/iwasm/aot/aot_runtime.h b/core/iwasm/aot/aot_runtime.h index 895823af..4358e307 100644 --- a/core/iwasm/aot/aot_runtime.h +++ b/core/iwasm/aot/aot_runtime.h @@ -92,30 +92,6 @@ typedef struct AOTModuleInstanceExtra { WASMModuleInstanceExtraCommon common; } AOTModuleInstanceExtra; -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) -/* clang-format off */ -typedef struct AOTUnwindInfo { - uint8 Version : 3; - uint8 Flags : 5; - uint8 SizeOfProlog; - uint8 CountOfCodes; - uint8 FrameRegister : 4; - uint8 FrameOffset : 4; - struct { - struct { - uint8 CodeOffset; - uint8 UnwindOp : 4; - uint8 OpInfo : 4; - }; - uint16 FrameOffset; - } UnwindCode[1]; -} AOTUnwindInfo; -/* clang-format on */ - -/* size of mov instruction and jmp instruction */ -#define PLT_ITEM_SIZE 12 -#endif - #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) typedef struct GOTItem { uint32 func_idx; @@ -211,14 +187,6 @@ typedef struct AOTModule { uint32 float_plt_count; #endif -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - /* dynamic function table to be added by RtlAddFunctionTable(), - used to unwind the call stack and register exception handler - for AOT functions */ - RUNTIME_FUNCTION *rtl_func_table; - bool rtl_func_table_registered; -#endif - #if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64) uint32 got_item_count; GOTItemList got_item_list; diff --git a/core/iwasm/aot/arch/aot_reloc_x86_64.c b/core/iwasm/aot/arch/aot_reloc_x86_64.c index 1bf5eb59..b5be2459 100644 --- a/core/iwasm/aot/arch/aot_reloc_x86_64.c +++ b/core/iwasm/aot/arch/aot_reloc_x86_64.c @@ -69,9 +69,6 @@ get_plt_table_size() { uint32 size = get_plt_item_size() * (sizeof(target_sym_map) / sizeof(SymbolMap)); -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - size += get_plt_item_size() + sizeof(AOTUnwindInfo); -#endif return size; } @@ -93,18 +90,6 @@ init_plt_table(uint8 *plt) *p++ = 0xE0; plt += get_plt_item_size(); } - -#if defined(OS_ENABLE_HW_BOUND_CHECK) && defined(BH_PLATFORM_WINDOWS) - p = plt; - /* mov exception_handler, rax */ - *p++ = 0x48; - *p++ = 0xB8; - *(uint64 *)p = 0; /*(uint64)(uintptr_t)aot_exception_handler;*/ - p += sizeof(uint64); - /* jmp rax */ - *p++ = 0xFF; - *p++ = 0xE0; -#endif } static bool diff --git a/core/iwasm/aot/iwasm_aot.cmake b/core/iwasm/aot/iwasm_aot.cmake index 8014f237..efff88dd 100644 --- a/core/iwasm/aot/iwasm_aot.cmake +++ b/core/iwasm/aot/iwasm_aot.cmake @@ -36,5 +36,49 @@ if (WAMR_BUILD_DEBUG_AOT EQUAL 1) file(GLOB debug_source ${IWASM_AOT_DIR}/debug/*.c) endif() -set (IWASM_AOT_SOURCE ${c_source_all} ${arch_source} ${debug_source}) +if ((WAMR_BUILD_TARGET STREQUAL "X86_64" OR WAMR_BUILD_TARGET STREQUAL "AMD_64") + AND (WAMR_BUILD_PLATFORM STREQUAL "windows") + AND (NOT WAMR_DISABLE_HW_BOUND_CHECK EQUAL 1)) + include(FetchContent) + FetchContent_Declare( + zycore + GIT_REPOSITORY https://github.com/zyantific/zycore-c.git + ) + FetchContent_GetProperties(zycore) + if (NOT zycore_POPULATED) + message ("-- Fetching zycore ..") + FetchContent_Populate(zycore) + include_directories("${zycore_SOURCE_DIR}/include") + include_directories("${zycore_BINARY_DIR}") + add_definitions(-DZYCORE_STATIC_BUILD=1) + add_subdirectory(${zycore_SOURCE_DIR} ${zycore_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zycore ${zycore_SOURCE_DIR}/src/*.c) + endif () + + FetchContent_Declare( + zydis + GIT_REPOSITORY https://github.com/zyantific/zydis.git + GIT_TAG e14a07895136182a5b53e181eec3b1c6e0b434de + ) + FetchContent_GetProperties(zydis) + if (NOT zydis_POPULATED) + message ("-- Fetching zydis ..") + FetchContent_Populate(zydis) + option(ZYDIS_FEATURE_ENCODER "" OFF) + option(ZYDIS_BUILD_TOOLS "" OFF) + option(ZYDIS_BUILD_EXAMPLES "" OFF) + option(ZYDIS_BUILD_MAN "" OFF) + option(ZYDIS_BUILD_DOXYGEN "" OFF) + include_directories("${zydis_BINARY_DIR}") + include_directories("${zydis_SOURCE_DIR}/include") + include_directories("${zydis_SOURCE_DIR}/src") + add_definitions(-DZYDIS_STATIC_BUILD=1) + add_subdirectory(${zydis_SOURCE_DIR} ${zydis_BINARY_DIR} EXCLUDE_FROM_ALL) + file (GLOB_RECURSE c_source_zydis ${zydis_SOURCE_DIR}/src/*.c) + endif () +endif () + + +set (IWASM_AOT_SOURCE ${c_source_all} ${arch_source} ${debug_source} + ${c_source_zycore} ${c_source_zydis}) diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index b1866215..09233f24 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -199,7 +199,90 @@ runtime_signal_handler(void *sig_addr) } } } -#else +#else /* else of BH_PLATFORM_WINDOWS */ + +#if WASM_ENABLE_AOT != 0 +#include + +static uint32 +decode_insn(uint8 *insn) +{ + uint8 *data = (uint8 *)insn; + uint32 length = 32; /* reserve enough size */ + + /* Initialize decoder context */ + ZydisDecoder decoder; + ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, + ZYDIS_STACK_WIDTH_64); + + /* Initialize formatter */ + ZydisFormatter formatter; + ZydisFormatterInit(&formatter, ZYDIS_FORMATTER_STYLE_INTEL); + + /* Loop over the instructions in our buffer */ + ZyanU64 runtime_address = (ZyanU64)(uintptr_t)data; + ZyanUSize offset = 0; + ZydisDecodedInstruction instruction; + ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT_VISIBLE]; + char buffer[256]; + + if (ZYAN_SUCCESS(ZydisDecoderDecodeFull( + &decoder, data + offset, length - offset, &instruction, operands, + ZYDIS_MAX_OPERAND_COUNT_VISIBLE, + ZYDIS_DFLAG_VISIBLE_OPERANDS_ONLY))) { + + /* Format & print the binary instruction structure to + human readable format */ + ZydisFormatterFormatInstruction(&formatter, &instruction, operands, + instruction.operand_count_visible, + buffer, sizeof(buffer), + runtime_address); + + /* Print current instruction */ + /* + os_printf("%012" PRIX64 " ", runtime_address); + puts(buffer); + */ + + return instruction.length; + } + + /* Decode failed */ + return 0; +} +#endif /* end of WASM_ENABLE_AOT != 0 */ + +static LONG +next_action(WASMModuleInstance *module_inst, EXCEPTION_POINTERS *exce_info) +{ +#if WASM_ENABLE_AOT != 0 + uint32 insn_size; +#endif + + if (module_inst->module_type == Wasm_Module_Bytecode + && module_inst->e->running_mode == Mode_Interp) { + /* Continue to search next exception handler for + interpreter mode as it can be caught by + `__try { .. } __except { .. }` sentences in + wasm_runtime.c */ + return EXCEPTION_CONTINUE_SEARCH; + } + +#if WASM_ENABLE_AOT != 0 + /* Skip current instruction and continue to run for AOT/JIT mode. + TODO: implement unwind support for AOT/JIT code in Windows platform */ + insn_size = decode_insn((uint8 *)exce_info->ContextRecord->Rip); + if (insn_size > 0) { + exce_info->ContextRecord->Rip += insn_size; + return EXCEPTION_CONTINUE_EXECUTION; + } +#endif + + /* return different value from EXCEPTION_CONTINUE_SEARCH (= 0) + and EXCEPTION_CONTINUE_EXECUTION (= -1) */ + return -2; +} + static LONG runtime_exception_handler(EXCEPTION_POINTERS *exce_info) { @@ -211,6 +294,7 @@ runtime_exception_handler(EXCEPTION_POINTERS *exce_info) uint8 *mapped_mem_start_addr = NULL; uint8 *mapped_mem_end_addr = NULL; uint32 page_size = os_getpagesize(); + LONG ret; if (exec_env_tls && exec_env_tls->handle == os_self_thread() && (jmpbuf_node = exec_env_tls->jmpbuf_stack_top)) { @@ -232,32 +316,19 @@ runtime_exception_handler(EXCEPTION_POINTERS *exce_info) the wasm func returns, the caller will check whether the exception is thrown and return to runtime. */ wasm_set_exception(module_inst, "out of bounds memory access"); - if (module_inst->module_type == Wasm_Module_Bytecode) { - /* Continue to search next exception handler for - interpreter mode as it can be caught by - `__try { .. } __except { .. }` sentences in - wasm_runtime.c */ - return EXCEPTION_CONTINUE_SEARCH; - } - else { - /* Skip current instruction and continue to run for - AOT mode. TODO: implement unwind support for AOT - code in Windows platform */ - exce_info->ContextRecord->Rip++; - return EXCEPTION_CONTINUE_EXECUTION; - } + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; } else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr && (uint8 *)sig_addr < exec_env_tls->exce_check_guard_page + page_size) { bh_assert(wasm_copy_exception(module_inst, NULL)); - if (module_inst->module_type == Wasm_Module_Bytecode) { - return EXCEPTION_CONTINUE_SEARCH; - } - else { - exce_info->ContextRecord->Rip++; - return EXCEPTION_CONTINUE_EXECUTION; - } + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; } } #if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 @@ -267,12 +338,10 @@ runtime_exception_handler(EXCEPTION_POINTERS *exce_info) whether the exception is thrown and return to runtime, and the damaged stack will be recovered by _resetstkoflw(). */ wasm_set_exception(module_inst, "native stack overflow"); - if (module_inst->module_type == Wasm_Module_Bytecode) { - return EXCEPTION_CONTINUE_SEARCH; - } - else { - return EXCEPTION_CONTINUE_EXECUTION; - } + ret = next_action(module_inst, exce_info); + if (ret == EXCEPTION_CONTINUE_SEARCH + || ret == EXCEPTION_CONTINUE_EXECUTION) + return ret; } #endif } diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index aa837a3c..413f4769 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -18,6 +18,17 @@ } \ } while (0) +static bool +is_win_platform(AOTCompContext *comp_ctx) +{ + char *triple = LLVMGetTargetMachineTriple(comp_ctx->target_machine); + + bh_assert(trip); + if (strstr(triple, "win32") || strstr(triple, "win")) + return true; + return false; +} + static bool create_func_return_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -458,7 +469,7 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Check whether exception was thrown when executing the function */ - if (comp_ctx->enable_bound_check + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) && !check_call_return(comp_ctx, func_ctx, res)) { return false; } @@ -696,7 +707,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; /* Check whether there was exception thrown when executing the function */ - if (comp_ctx->enable_bound_check + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) && !check_call_return(comp_ctx, func_ctx, res)) goto fail; } @@ -849,7 +860,8 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Check whether there was exception thrown when executing the function */ - if (!tail_call && comp_ctx->enable_bound_check + if (!tail_call + && (comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) && !check_exception_thrown(comp_ctx, func_ctx)) goto fail; } @@ -1431,7 +1443,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; /* Check whether exception was thrown when executing the function */ - if (comp_ctx->enable_bound_check + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) && !check_call_return(comp_ctx, func_ctx, res)) goto fail; @@ -1483,7 +1495,7 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } /* Check whether exception was thrown when executing the function */ - if (comp_ctx->enable_bound_check + if ((comp_ctx->enable_bound_check || is_win_platform(comp_ctx)) && !check_exception_thrown(comp_ctx, func_ctx)) goto fail; From 0e12cdec487e1c7c55d6212c82e7534bb5f2199e Mon Sep 17 00:00:00 2001 From: Maks Litskevich Date: Fri, 18 Aug 2023 03:40:05 +0100 Subject: [PATCH 25/34] Add sample pre-commit hook (#2470) Using pre-commit hooks helps us to decrease CI load and improve developers' experience. --- ci/pre_commit_hook_sample | 32 ++++++++++++++++++++++++++++++++ ci/setup.sh | 15 +++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 ci/pre_commit_hook_sample create mode 100755 ci/setup.sh diff --git a/ci/pre_commit_hook_sample b/ci/pre_commit_hook_sample new file mode 100755 index 00000000..682e7894 --- /dev/null +++ b/ci/pre_commit_hook_sample @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This is a sample of pre-commit hook that can be used to make your code fit the WAMR CI code style requirements. +# You need to have clang-format-12 installed to use this hook. +# To add this pre-commit hook, copy it to /.git/hooks/pre-commit +# (you don't need any extensions here) + +# Function to check if a file has a C or C++ extension + +is_c_or_cpp_file() { + file="$1" + if [[ "$filename" =~ \.(h|c|cpp)$ ]]; then + return 0 + else + return 1 + fi +} + +# Loop through staged files and apply command "abc" to C and C++ files +for staged_file in $(git diff --cached --name-only); do + if is_c_or_cpp_file "$staged_file"; then + clang-format-12 -Werror --style file --dry-run "$staged_file" 2>/dev/null + if [ $? -ne 0 ]; then + echo "Issues are found in $staged_file. Applying the fix" + clang-format-12 --style file -i "$staged_file" + fi + git add "$staged_file" # Add the modified file back to staging + fi +done diff --git a/ci/setup.sh b/ci/setup.sh new file mode 100755 index 00000000..ebdf73af --- /dev/null +++ b/ci/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Copyright (C) 2023 Amazon.com Inc. or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This script executes some commands to make your onboarding with WAMR easier. +# For example, setting pre-commit hook that will make your code complaint with the +# code style requirements checked in WAMR CI + +echo "Copy the pre-commit hook to your hooks folder" +cp pre_commit_hook_sample ../.git/hooks/pre-commit + +# Feel free to propose your commands to this script to make developing WAMR easier + +echo "Setup is done" From a2f76cf93c271b02db1456bbb05134b08809a883 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Fri, 18 Aug 2023 13:57:23 +0800 Subject: [PATCH 26/34] Fix typo in aot_emit_aot_file.c (#2478) --- core/iwasm/compilation/aot_emit_aot_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/compilation/aot_emit_aot_file.c b/core/iwasm/compilation/aot_emit_aot_file.c index 1b1e4403..62218775 100644 --- a/core/iwasm/compilation/aot_emit_aot_file.c +++ b/core/iwasm/compilation/aot_emit_aot_file.c @@ -1825,7 +1825,7 @@ aot_emit_text_section(uint8 *buf, uint8 *buf_end, uint32 *p_offset, == 4 /* IMAGE_REL_AMD64_REL32 */) || (obj_data->target_info.bin_type == 4 /* AOT_COFF32_BIN_TYPE */ - && obj_data->target_info.bin_type + && relocation->relocation_type == 20 /* IMAGE_REL_I386_REL32 */))) { uint32 func_idx = atoi(relocation->symbol_name From f0632edc37543aa1493efb9dbcd836eb58dbed23 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Mon, 21 Aug 2023 19:43:28 +0800 Subject: [PATCH 27/34] Clone the input binary during wasm_module_validate (#2483) --- core/iwasm/common/wasm_c_api.c | 21 +++++++++++++++++---- samples/wasm-c-api/src/hello.c | 6 ++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index a595a626..afb38cc9 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -2290,8 +2290,10 @@ quit: bool wasm_module_validate(wasm_store_t *store, const wasm_byte_vec_t *binary) { + wasm_byte_vec_t local_binary = { 0 }; struct WASMModuleCommon *module_rt; char error_buf[128] = { 0 }; + bool ret; bh_assert(singleton_engine); @@ -2300,15 +2302,26 @@ wasm_module_validate(wasm_store_t *store, const wasm_byte_vec_t *binary) return false; } - if ((module_rt = wasm_runtime_load((uint8 *)binary->data, - (uint32)binary->size, error_buf, 128))) { + /* make a copy of binary */ + wasm_byte_vec_new_uninitialized(&local_binary, binary->size); + if (binary->size && !local_binary.data) + return false; + + wasm_byte_vec_copy(&local_binary, binary); + + module_rt = wasm_runtime_load((uint8 *)local_binary.data, + (uint32)local_binary.size, error_buf, 128); + wasm_byte_vec_delete(&local_binary); + if (module_rt) { wasm_runtime_unload(module_rt); - return true; + ret = true; } else { + ret = false; LOG_VERBOSE(error_buf); - return false; } + + return ret; } static void diff --git a/samples/wasm-c-api/src/hello.c b/samples/wasm-c-api/src/hello.c index 3ebea87b..966e141d 100644 --- a/samples/wasm-c-api/src/hello.c +++ b/samples/wasm-c-api/src/hello.c @@ -65,6 +65,12 @@ int main(int argc, const char* argv[]) { } fclose(file); + if (!wasm_module_validate(store, &binary)) { + printf("> Error validate module!\n"); + wasm_byte_vec_delete(&binary); + return 1; + } + // Compile. printf("Compiling module...\n"); own wasm_module_t* module = wasm_module_new(store, &binary); From 59bedecc5eb718e03980f4d1ac27749d30bd33a6 Mon Sep 17 00:00:00 2001 From: tonibofarull Date: Tue, 22 Aug 2023 02:50:26 +0200 Subject: [PATCH 28/34] Fix set wasi args order (#2485) set_wasi_args should be called before module instantiation. --- language-bindings/python/src/wamr/wamrapi/wamr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/language-bindings/python/src/wamr/wamrapi/wamr.py b/language-bindings/python/src/wamr/wamrapi/wamr.py index 74cbad9f..1bd6e547 100644 --- a/language-bindings/python/src/wamr/wamrapi/wamr.py +++ b/language-bindings/python/src/wamr/wamrapi/wamr.py @@ -131,12 +131,12 @@ class Instance: ): # Store module ensures GC does not remove it self.module = module + if dir_list: + self._set_wasi_args(module, dir_list) if preinitialized_module_inst is None: self.module_inst = self._create_module_inst(module, stack_size, heap_size) else: self.module_inst = preinitialized_module_inst - if dir_list: - self._set_wasi_args(module, dir_list) def __del__(self): print("deleting Instance") From 7f996a4559fe26dae57150c9bff99a04dad61d31 Mon Sep 17 00:00:00 2001 From: Marcin Kolny Date: Tue, 22 Aug 2023 01:55:36 +0100 Subject: [PATCH 29/34] Update version number and release notes (#2378) Update the version number to 1.2.3 and update the release notes. --- RELEASE_NOTES.md | 136 +++++++++++++++++++++++++++++++++++++++++++++++ core/version.h | 2 +- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d6308ce6..5c7f598d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,139 @@ +## WAMR-1.2.3 + +### Breaking Changes +- Increase default native stack size (#2332) + +### New Features +- Implement the segue optimization for LLVM AOT/JIT (#2230) +- Implement AOT static PGO (#2243) +- Enable static PGO for Linux SGX (#2270) +- Add Rust Formatters to Debugger (Vector, Map etc.) (#2219) + +### Bug Fixes +- The Python language-binding needs python>=3.9 (#2228) +- aot_compile_op_call: Remove a wrong optimization (#2233) +- Fix typo in samples/ref-types (#2236) +- Update thread proposal ignore cases (#2246) +- Disable writting GS register on linux-sgx platform (#2255) +- Fix compile error of wamrc with llvm-13/llvm-14 (#2261) +- aot/jit: Set module layout (#2260) +- Fix build error with LLVM 16 (#2259) +- spec-test-script: Disable conversions.wast on i386 (#2269) +- Fix a heap corruption bug in ems realloc (#2279) +- Fix fast-interp issue of LAST_OP_OUTPUT_I32/64 check (#2295) +- Fix wamrc build issues with LLVM 13 and LLVM 16 (#2313) +- aot: Move stack_sizes table to a dedicated section (#2317) +- product-mini/platforms/linux: Mark vmlib POSITION_INDEPENDENT_CODE (#2323) +- aot: Avoid possible relocations around "stack_sizes" for XIP mode (#2322) +- Avoid switch lowering to lookup tables for XIP (#2339) +- Fix typo in zephyr's Dockerfile.old (#2354) +- Fix typo (dwarf) in the codebase (#2367) +- Implement suspend flags as atomic variable (#2361) +- Fix llvm jit failed to lookup aot_stack_sizes symbol issue (#2384) +- Fix some check issues on table operations (#2392) +- Fix ExpandMemoryOpPass doesn't work properly (#2399) +- Fix non-builtin BH_ATOMIC_32_FETCH_OR and BH_ATOMIC_32_FETCH_AND (#2400) +- Fix wasi-sockets tests (#2389) +- Fix result arity check on select_t opcode (#2406) +- Re-organize intrinsics in aot_reloc_riscv.c to fix some FPU issues (#2414) +- Fix lib-pthread issues (#2410) +- Fix typo in test_wamr.sh (#2421) +- Fix memory sharing (#2415) +- wasm_export.h: Fix struct wasm_val_t (#2435) +- Fix typos in wamrc print_help() (#2442) +- iwasm: Fix native lib cleanup after error occurs (#2443) +- Correct --heap-size option in messages (#2458) +- wasm_instantiate: Fix a potential integer overflow issue (#2459) +- Fix windows link error and clear windows warnings (#2463) +- aot: Disable musttail for mips (#2457) +- Fix opcode overwrite issue in fast interp (#2476) +- wamrc: Fix windows relocation to `aot_func_internal#n` (#2474) +- Fix windows AOT hw bound check (#2475) +- Fix typo in aot_emit_aot_file.c (#2478) + +### Enhancements +- A few changes related to WAMRC_LLC_COMPILER (#2218) +- Enhance linux-sgx CI (#2102) +- Add asan and ubsan to WAMR CI (#2161) +- Update doc on WAMR_DISABLE_HW_BOUND_CHECK 32-bit (#2262) +- wamrc: Add an incompatibility note in the help message (#2276) +- Add cmake variable to disable writing gs register (#2284) +- Make hmu_tree_node 4 byte aligned to reduce compiler warning (#2268) +- Appease unused warning on min_uint64 (#2277) +- Fix format warning by PRIu32 in [wasm|aot] dump call stack (#2251) +- Fix a compile warning due to missing include (#2293) +- Fix dockerfile linter warnings (#2291) +- Enable windows x86-32 AOT relocations (#2285) +- wamr-ide: Add vscode extension tests (#2292) +- AOT/JIT native stack bound check improvement (#2244) +- Add retries to flaky step in nightly run CI (#2306) +- Use system libuv if available (#1861) +- wasi-nn: Simplify cmake and headers' location (#2308) +- wasi-nn: Improve tests paths for local dev (#2309) +- aot: Implement a few more relocation types for riscv (#2318) +- wasi-nn: Add support of wasi-nn as shared lib (#2310) +- Add a few more assertions on structures to which aot abi is sensitive (#2326) +- Fix sanitizer errors in posix socket (#2331) +- Add "--xip" option for wamrc (#2336) +- Add "--enable-llvm-passes=" option to wamrc (#2335) +- Make memory access boundary check behavior configurable (#2289) +- Migrate ExpandMemoryOpPass to llvm new pass manager (#2334) +- Allow defining hints without exact socket type or address family (#2337) +- wamrc: Warn on text relocations for XIP (#2340) +- Add scripts to validate lldb source debugger (#2150) +- Add docker file to fix Zephy ESP32 linking issue (#2314) +- Add "--native-lib=" option to wamrc (#2342) +- Fix unused warnings on disable_bounds_checks (#2347) +- Add "--enable-builtin-intrinsics=" option to wamrc (#2341) +- nuttx: Add a kconfig for wasi-threads (#2343) +- iwasm: Disable app heap by default if wasi is enabled (#2346) +- Fix some static scan issues (#2362) +- Bring up WAMR on esp32-s3 device (#2348) +- ESP-IDF platform supports to load AOT to PSRAM and run it (#2385) +- Add hadolint CI for Dockerfile linting (#2387) +- Move generic parts of wasm_suspend_flags.h to bh_atomic.h (#2393) +- bh_atomic.h: Add comments (#2398) +- bh_atomic.h: Add BH_ATOMIC_32_FETCH_ADD/BH_ATOMIC_32_FETCH_SUB (#2408) +- Update libuv version to v1.46.0 (#2405) +- Remove a few unused functions (#2409) +- Add initial stress test (#2364) +- Move wasm_runtime_destroy_wasi and wasi_nn_destroy calls together (#2418) +- embed_wamr.md: Improvements about threads (#2420) +- Add runtime inited checks in Enclave command handlings to improve security (#2416) +- Add some relocation symbols for xtensa target (#2422) +- Remove unnecessary and extra zero length check in mem functions' macro (#2428) +- Introduce WASMModuleInstanceExtraCommon (#2429) +- Strip static and shared libraries of iwasm to reduce the binary size (#2431) +- Auto-check wrgsbase in cmake script (#2437) +- iwasm: call native lib init/deinit if exists (#2439) +- wasi-nn: Support uint8 quantized networks (#2433) +- Implement `wasm_externref_objdel` and `wasm_externref_set_cleanup` (#2455) +- wasi-nn: Improve TPU support (#2447) +- wamr-python: Enable debugging WASM and grant dir access (#2449) +- Build wasi-libc from source in WAMR CI (#2465) +- wamrc: More friendly to print help info (#2451) +- Add another wamr test (#2411) +- Fix issues reported by Coverity and clear windows warnings (#2467) +- Clone the input binary during wasm_module_validate (#2483) + +### Others +- Nuttx CI: Ignore the expired certificate for riscv gcc toolchain (#2222) +- core/iwasm/compilation: constify a bit (#2223) +- Bump requests from 2.28.2 to 2.31.0 in /build-scripts (#2229) +- dwarf_extractor: Constify a bit (#2278) +- AOTFuncContext: Remove a stale comment (#2283) +- Add performance tunning document (#2286) +- Reduce CI jobs number (#2296) +- CI: Update used node version to 16 (#2303) +- Update Docker image for latest version of external libraries & tools (#2374) +- Upgrade cJSON version to v1.7.16 (#2404) +- Upgrade XNNPACK workload (#2394) +- Build more benchmarks in workload XNNPACK (#2417) +- Upgrade SGX-RA integration for 0.1.2 and Ubuntu 20.04 (#2454) +- Add sample pre-commit hook (#2470) + +--- + ## WAMR-1.2.2 ### Breaking Changes diff --git a/core/version.h b/core/version.h index de24b30b..925779b3 100644 --- a/core/version.h +++ b/core/version.h @@ -7,5 +7,5 @@ #define _WAMR_VERSION_H_ #define WAMR_VERSION_MAJOR 1 #define WAMR_VERSION_MINOR 2 -#define WAMR_VERSION_PATCH 2 +#define WAMR_VERSION_PATCH 3 #endif From 051dfbbb9e4bfa2b26b220836a78192021762112 Mon Sep 17 00:00:00 2001 From: Huang Qi Date: Tue, 22 Aug 2023 11:50:11 +0800 Subject: [PATCH 30/34] Fix a typo in is_win_platform (#2490) --- core/iwasm/compilation/aot_emit_function.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c index 413f4769..def3f7d0 100644 --- a/core/iwasm/compilation/aot_emit_function.c +++ b/core/iwasm/compilation/aot_emit_function.c @@ -23,7 +23,7 @@ is_win_platform(AOTCompContext *comp_ctx) { char *triple = LLVMGetTargetMachineTriple(comp_ctx->target_machine); - bh_assert(trip); + bh_assert(triple); if (strstr(triple, "win32") || strstr(triple, "win")) return true; return false; From 2b30bb0fdac0a403c27c579c64a46af80068d479 Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Tue, 22 Aug 2023 13:57:23 +0800 Subject: [PATCH 31/34] Upgrade swig to 4.1 (#2491) Upgrade to sig@4.1 when building wamr-lldb on MacOS since swig@3 had been disabled: https://formulae.brew.sh/formula/swig --- .github/workflows/build_wamr_lldb.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_wamr_lldb.yml b/.github/workflows/build_wamr_lldb.yml index e0f9595c..b7971aa8 100644 --- a/.github/workflows/build_wamr_lldb.yml +++ b/.github/workflows/build_wamr_lldb.yml @@ -27,7 +27,7 @@ on: type: string required: false default: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz" - + jobs: try_reuse: @@ -49,7 +49,7 @@ jobs: PYTHON_MACOS_STANDALONE_BUILD: https://github.com/indygreg/python-build-standalone/releases/download/20230507/cpython-3.10.11+20230507-x86_64-apple-darwin-install_only.tar.gz steps: - uses: actions/checkout@v3 - + - name: download and install wasi-sdk run: | cd /opt @@ -84,8 +84,8 @@ jobs: if: steps.lldb_build_cache.outputs.cache-hit != 'true' && contains(inputs.runner, 'macos') run: | brew remove swig - brew install swig@3 cmake ninja libedit - brew link --overwrite swig@3 + brew install swig@4.1 cmake ninja libedit + brew link --overwrite swig@4.1 sudo rm -rf /Library/Developer/CommandLineTools - name: install utils ubuntu @@ -205,7 +205,7 @@ jobs: mkdir -p wamr-lldb/lib cp build/bin/lldb* wamr-lldb/bin cp lldb/tools/lldb-vscode/package.json wamr-lldb - cp -r lldb/tools/lldb-vscode/syntaxes/ wamr-lldb + cp -r lldb/tools/lldb-vscode/syntaxes/ wamr-lldb working-directory: core/deps/llvm-project - name: pack ubuntu specific libraries @@ -226,7 +226,7 @@ jobs: cp -R ../python/lib/python* wamr-lldb/lib cp ../python/lib/libpython*.dylib wamr-lldb/lib install_name_tool -change /install/lib/libpython${{ env.PYTHON_VERSION }}.dylib @rpath/libpython${{ env.PYTHON_VERSION }}.dylib wamr-lldb/lib/liblldb.*.dylib - # Patch path of python library -> https://github.com/indygreg/python-build-standalone/blob/85923ca3911784e6978b85d56e06e9ae75cb2dc4/docs/quirks.rst?plain=1#L412-L446 + # Patch path of python library -> https://github.com/indygreg/python-build-standalone/blob/85923ca3911784e6978b85d56e06e9ae75cb2dc4/docs/quirks.rst?plain=1#L412-L446 working-directory: core/deps/llvm-project - name: compress the binary From 377c3d7a38aeb9cb26be231408432cc2369300eb Mon Sep 17 00:00:00 2001 From: Xu Jun Date: Tue, 22 Aug 2023 14:24:42 +0800 Subject: [PATCH 32/34] Auto increase port number in lldb validation script (#2492) --- ci/validate_lldb.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/validate_lldb.py b/ci/validate_lldb.py index 0bf743c0..1b431ada 100755 --- a/ci/validate_lldb.py +++ b/ci/validate_lldb.py @@ -32,7 +32,6 @@ parser.add_argument( options = parser.parse_args() -lldb_command_prologue = f'{options.lldb} -o "process connect -p wasm connect://127.0.0.1:{options.port}"' lldb_command_epilogue = '-o q' test_cases = { @@ -64,8 +63,13 @@ def print_process_output(p): print("Failed to get process output") # Step2: Launch WAMR in debug mode and validate lldb commands -wamr_cmd = f'{options.wamr} -g=127.0.0.1:{options.port} {WASM_OUT_FILE}' + +iteration = 0 for case, cmd in test_cases.items(): + lldb_command_prologue = f'{options.lldb} -o "process connect -p wasm connect://127.0.0.1:{int(options.port) + iteration}"' + wamr_cmd = f'{options.wamr} -g=127.0.0.1:{int(options.port) + iteration} {WASM_OUT_FILE}' + iteration += 1 + has_error = False print(f'validating case [{case}] ...', end='', flush=True) lldb_cmd = f'{lldb_command_prologue} {cmd} {lldb_command_epilogue}' From 858d1dd69cad4d672c791f0c5af38dd3d5a0f936 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 23 Aug 2023 12:35:46 +0800 Subject: [PATCH 33/34] Fix duplicate vector init in wasm_module_validate (#2498) --- core/iwasm/common/wasm_c_api.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/iwasm/common/wasm_c_api.c b/core/iwasm/common/wasm_c_api.c index afb38cc9..2da73387 100644 --- a/core/iwasm/common/wasm_c_api.c +++ b/core/iwasm/common/wasm_c_api.c @@ -2303,12 +2303,11 @@ wasm_module_validate(wasm_store_t *store, const wasm_byte_vec_t *binary) } /* make a copy of binary */ - wasm_byte_vec_new_uninitialized(&local_binary, binary->size); + wasm_byte_vec_copy(&local_binary, binary); + if (binary->size && !local_binary.data) return false; - wasm_byte_vec_copy(&local_binary, binary); - module_rt = wasm_runtime_load((uint8 *)local_binary.data, (uint32)local_binary.size, error_buf, 128); wasm_byte_vec_delete(&local_binary); From fa2f29fd8ab3f72e3efb551d19cb6b214ead91f8 Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 23 Aug 2023 12:56:11 +0800 Subject: [PATCH 34/34] Fix issues in releasing wamr-lldb (#2497) - Update lldb patch due to swig was upgraded to 4.1 in macos - Export LD_LIBRARY_PATH for searching libpython3.10.so when validating wamr-lldb in Ubuntu-20.04 - Rename lldb-wasm.patch to lldb_wasm.path --- .github/workflows/build_wamr_lldb.yml | 6 +- .../compilation_on_android_ubuntu.yml | 4 +- .../{lldb-wasm.patch => lldb_wasm.patch} | 155 +++++++++++------- doc/source_debugging.md | 2 +- 4 files changed, 103 insertions(+), 64 deletions(-) rename build-scripts/{lldb-wasm.patch => lldb_wasm.patch} (98%) diff --git a/.github/workflows/build_wamr_lldb.yml b/.github/workflows/build_wamr_lldb.yml index b7971aa8..1a73750a 100644 --- a/.github/workflows/build_wamr_lldb.yml +++ b/.github/workflows/build_wamr_lldb.yml @@ -28,14 +28,13 @@ on: required: false default: "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-linux.tar.gz" - jobs: try_reuse: uses: ./.github/workflows/reuse_latest_release_binaries.yml with: binary_name_stem: "wamr-lldb-${{ inputs.ver_num }}-${{ inputs.arch }}-${{ inputs.runner }}" last_commit: "ea63ba4bd010c2285623ad4acc0262a4d63bcfea" - the_path: "./build-scripts/lldb-wasm.patch" + the_path: "./build-scripts/lldb_wasm.patch" upload_url: ${{ inputs.upload_url }} build: @@ -107,7 +106,7 @@ jobs: git init git config user.email "action@github.com" git config user.name "github action" - git apply ../../../build-scripts/lldb-wasm.patch + git apply ../../../build-scripts/lldb_wasm.patch working-directory: core/deps/llvm-project - name: get stand-alone python ubuntu @@ -163,6 +162,7 @@ jobs: mkdir -p wamr-debug cmake -S product-mini/platforms/linux -B wamr-debug -DWAMR_BUILD_DEBUG_INTERP=1 cmake --build wamr-debug --parallel $(nproc) + export LD_LIBRARY_PATH=$(pwd)/core/deps/python/lib:${LD_LIBRARY_PATH} python3 ci/validate_lldb.py --port 1239 --lldb core/deps/wamr-lldb/bin/lldb --wamr wamr-debug/iwasm --verbose working-directory: . diff --git a/.github/workflows/compilation_on_android_ubuntu.yml b/.github/workflows/compilation_on_android_ubuntu.yml index bbb2c4b3..b7ac084e 100644 --- a/.github/workflows/compilation_on_android_ubuntu.yml +++ b/.github/workflows/compilation_on_android_ubuntu.yml @@ -594,7 +594,7 @@ jobs: cache-name: cache-lldb-vscode with: path: test-tools/wamr-ide/VSCode-Extension/resource/debug/linux - key: ${{ env.cache-name }}-${{ hashFiles('build-scripts/lldb-wasm.patch') }}-${{ env.PYTHON_UBUNTU_STANDALONE_BUILD }} + key: ${{ env.cache-name }}-${{ hashFiles('build-scripts/lldb_wasm.patch') }}-${{ env.PYTHON_UBUNTU_STANDALONE_BUILD }} - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} name: get stand-alone python ubuntu @@ -617,7 +617,7 @@ jobs: git init git config user.email "action@github.com" git config user.name "github action" - git apply ../../../build-scripts/lldb-wasm.patch + git apply ../../../build-scripts/lldb_wasm.patch working-directory: core/deps/llvm-project - if: ${{ steps.cache-lldb.outputs.cache-hit != 'true' }} diff --git a/build-scripts/lldb-wasm.patch b/build-scripts/lldb_wasm.patch similarity index 98% rename from build-scripts/lldb-wasm.patch rename to build-scripts/lldb_wasm.patch index d6044e3a..83221ef6 100644 --- a/build-scripts/lldb-wasm.patch +++ b/build-scripts/lldb_wasm.patch @@ -1,5 +1,44 @@ +diff --git a/lldb/bindings/CMakeLists.txt b/lldb/bindings/CMakeLists.txt +index 9759b069fdc4..25b427f8bcf2 100644 +--- a/lldb/bindings/CMakeLists.txt ++++ b/lldb/bindings/CMakeLists.txt +@@ -26,8 +26,6 @@ set(SWIG_COMMON_FLAGS + -features autodoc + -I${LLDB_SOURCE_DIR}/include + -I${CMAKE_CURRENT_SOURCE_DIR} +- -D__STDC_LIMIT_MACROS +- -D__STDC_CONSTANT_MACROS + ${DARWIN_EXTRAS} + ) + +diff --git a/lldb/bindings/interfaces.swig b/lldb/bindings/interfaces.swig +index c9a6d0f06056..021c7683d170 100644 +--- a/lldb/bindings/interfaces.swig ++++ b/lldb/bindings/interfaces.swig +@@ -1,8 +1,5 @@ + /* Various liblldb typedefs that SWIG needs to know about. */ + #define __extension__ /* Undefine GCC keyword to make Swig happy when processing glibc's stdint.h. */ +-/* The ISO C99 standard specifies that in C++ implementations limit macros such +- as INT32_MAX should only be defined if __STDC_LIMIT_MACROS is. */ +-#define __STDC_LIMIT_MACROS + %include "stdint.i" + + %include "lldb/lldb-defines.h" +diff --git a/lldb/bindings/python/python-typemaps.swig b/lldb/bindings/python/python-typemaps.swig +index b1ace4ff3b1e..5f8f4aa678c4 100644 +--- a/lldb/bindings/python/python-typemaps.swig ++++ b/lldb/bindings/python/python-typemaps.swig +@@ -439,7 +439,7 @@ bool SetNumberFromPyObject(double &number, PyObject *obj) { + + %typemap(out) lldb::FileSP { + $result = nullptr; +- lldb::FileSP &sp = $1; ++ const lldb::FileSP &sp = $1; + if (sp) { + PythonFile pyfile = unwrapOrSetPythonException(PythonFile::FromFile(*sp)); + if (!pyfile.IsValid()) diff --git a/lldb/include/lldb/Breakpoint/Breakpoint.h b/lldb/include/lldb/Breakpoint/Breakpoint.h -index f2e2a0d22..426d1129b 100644 +index f2e2a0d22784..426d1129bd10 100644 --- a/lldb/include/lldb/Breakpoint/Breakpoint.h +++ b/lldb/include/lldb/Breakpoint/Breakpoint.h @@ -9,6 +9,7 @@ @@ -11,7 +50,7 @@ index f2e2a0d22..426d1129b 100644 #include #include diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h -index dd7100c46..97d70daad 100644 +index dd7100c4616c..97d70daadbdc 100644 --- a/lldb/include/lldb/Core/Module.h +++ b/lldb/include/lldb/Core/Module.h @@ -41,6 +41,7 @@ @@ -41,7 +80,7 @@ index dd7100c46..97d70daad 100644 /// /// Tries to resolve \a vm_addr as a file address (if \a diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h -index be91929c6..8d876fc1f 100644 +index be91929c62e1..8d876fc1fa2f 100644 --- a/lldb/include/lldb/Core/PluginManager.h +++ b/lldb/include/lldb/Core/PluginManager.h @@ -508,6 +508,17 @@ public: @@ -64,7 +103,7 @@ index be91929c6..8d876fc1f 100644 } // namespace lldb_private diff --git a/lldb/include/lldb/Expression/DWARFEvaluator.h b/lldb/include/lldb/Expression/DWARFEvaluator.h new file mode 100644 -index 000000000..6811cbeae +index 000000000000..6811cbeae3d3 --- /dev/null +++ b/lldb/include/lldb/Expression/DWARFEvaluator.h @@ -0,0 +1,110 @@ @@ -180,7 +219,7 @@ index 000000000..6811cbeae +#endif // LLDB_EXPRESSION_DWARFEVALUATOR_H diff --git a/lldb/include/lldb/Expression/DWARFEvaluatorFactory.h b/lldb/include/lldb/Expression/DWARFEvaluatorFactory.h new file mode 100644 -index 000000000..f3b496c58 +index 000000000000..f3b496c580e4 --- /dev/null +++ b/lldb/include/lldb/Expression/DWARFEvaluatorFactory.h @@ -0,0 +1,56 @@ @@ -241,7 +280,7 @@ index 000000000..f3b496c58 + +#endif // LLDB_EXPRESSION_DWARFEVALUATORFACTORY_H diff --git a/lldb/include/lldb/Expression/DWARFExpression.h b/lldb/include/lldb/Expression/DWARFExpression.h -index 1490ac2d6..35c741d4e 100644 +index 1490ac2d614a..35c741d4e6ba 100644 --- a/lldb/include/lldb/Expression/DWARFExpression.h +++ b/lldb/include/lldb/Expression/DWARFExpression.h @@ -120,6 +120,10 @@ public: @@ -275,7 +314,7 @@ index 1490ac2d6..35c741d4e 100644 GetLocationExpression(lldb::addr_t load_function_start, lldb::addr_t addr) const; diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h -index aaa2470d2..c15f2db52 100644 +index aaa2470d2931..c15f2db52fbc 100644 --- a/lldb/include/lldb/Target/Process.h +++ b/lldb/include/lldb/Target/Process.h @@ -1434,7 +1434,7 @@ public: @@ -288,7 +327,7 @@ index aaa2470d2..c15f2db52 100644 /// Read of memory from a process. /// diff --git a/lldb/include/lldb/Target/ProcessTrace.h b/lldb/include/lldb/Target/ProcessTrace.h -index 7b9d6b13d..9525fc975 100644 +index 7b9d6b13dd6f..9525fc9750fd 100644 --- a/lldb/include/lldb/Target/ProcessTrace.h +++ b/lldb/include/lldb/Target/ProcessTrace.h @@ -59,7 +59,7 @@ public: @@ -301,7 +340,7 @@ index 7b9d6b13d..9525fc975 100644 size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, Status &error) override; diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h -index ad5298151..5a3c0b27a 100644 +index ad5298151e4a..5a3c0b27a738 100644 --- a/lldb/include/lldb/lldb-forward.h +++ b/lldb/include/lldb/lldb-forward.h @@ -74,6 +74,7 @@ class Disassembler; @@ -313,7 +352,7 @@ index ad5298151..5a3c0b27a 100644 class EmulateInstruction; class Environment; diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h -index 2ed083ec8..f4d500d19 100644 +index 2ed083ec8ae9..f4d500d198e8 100644 --- a/lldb/include/lldb/lldb-private-interfaces.h +++ b/lldb/include/lldb/lldb-private-interfaces.h @@ -113,6 +113,8 @@ typedef lldb::REPLSP (*REPLCreateInstance)(Status &error, @@ -326,7 +365,7 @@ index 2ed083ec8..f4d500d19 100644 /// \{ typedef llvm::Expected (*TraceCreateInstanceForSessionFile)( diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp -index 19c97be15..1647f93ec 100644 +index 19c97be15066..1647f93ec4f3 100644 --- a/lldb/source/Core/Module.cpp +++ b/lldb/source/Core/Module.cpp @@ -16,6 +16,7 @@ @@ -348,7 +387,7 @@ index 19c97be15..1647f93ec 100644 + return m_dwarf_evaluator_factory.get(); +} diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp -index fcaa868b0..59a404d4a 100644 +index fcaa868b083e..59a404d4a7e1 100644 --- a/lldb/source/Core/PluginManager.cpp +++ b/lldb/source/Core/PluginManager.cpp @@ -1597,3 +1597,32 @@ bool PluginManager::CreateSettingForStructuredDataPlugin( @@ -385,7 +424,7 @@ index fcaa868b0..59a404d4a 100644 + return GetDWARFEvaluatorFactoryInstances().GetCallbackAtIndex(idx); +} diff --git a/lldb/source/Core/Value.cpp b/lldb/source/Core/Value.cpp -index fb57c0fed..f92d6a54d 100644 +index fb57c0fedf04..f92d6a54de94 100644 --- a/lldb/source/Core/Value.cpp +++ b/lldb/source/Core/Value.cpp @@ -538,7 +538,7 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data, @@ -398,7 +437,7 @@ index fb57c0fed..f92d6a54d 100644 error.SetErrorStringWithFormat( "read memory from 0x%" PRIx64 " failed (%u of %u bytes read)", diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp -index 9c1ba99da..b15b214b2 100644 +index 9c1ba99da1d0..b15b214b2a2f 100644 --- a/lldb/source/Core/ValueObject.cpp +++ b/lldb/source/Core/ValueObject.cpp @@ -735,7 +735,7 @@ size_t ValueObject::GetPointeeData(DataExtractor &data, uint32_t item_idx, @@ -411,7 +450,7 @@ index 9c1ba99da..b15b214b2 100644 data.SetData(data_sp); return bytes_read; diff --git a/lldb/source/Expression/CMakeLists.txt b/lldb/source/Expression/CMakeLists.txt -index bf94361dd..4e76d547a 100644 +index bf94361dd6c1..4e76d547aeaf 100644 --- a/lldb/source/Expression/CMakeLists.txt +++ b/lldb/source/Expression/CMakeLists.txt @@ -1,5 +1,7 @@ @@ -424,7 +463,7 @@ index bf94361dd..4e76d547a 100644 ExpressionVariable.cpp diff --git a/lldb/source/Expression/DWARFEvaluator.cpp b/lldb/source/Expression/DWARFEvaluator.cpp new file mode 100644 -index 000000000..06107e136 +index 000000000000..06107e136197 --- /dev/null +++ b/lldb/source/Expression/DWARFEvaluator.cpp @@ -0,0 +1,1952 @@ @@ -2382,7 +2421,7 @@ index 000000000..06107e136 +} diff --git a/lldb/source/Expression/DWARFEvaluatorFactory.cpp b/lldb/source/Expression/DWARFEvaluatorFactory.cpp new file mode 100644 -index 000000000..c06126412 +index 000000000000..c0612641204a --- /dev/null +++ b/lldb/source/Expression/DWARFEvaluatorFactory.cpp @@ -0,0 +1,57 @@ @@ -2444,7 +2483,7 @@ index 000000000..c06126412 + object_address_ptr); +} diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp -index a10546c1d..4d13e4642 100644 +index a10546c1deae..4d13e4642af3 100644 --- a/lldb/source/Expression/DWARFExpression.cpp +++ b/lldb/source/Expression/DWARFExpression.cpp @@ -15,6 +15,8 @@ @@ -4261,7 +4300,7 @@ index a10546c1d..4d13e4642 100644 static DataExtractor ToDataExtractor(const llvm::DWARFLocationExpression &loc, diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp -index 00e9ccb76..2137a1ac8 100644 +index 00e9ccb762c3..2137a1ac8324 100644 --- a/lldb/source/Interpreter/CommandInterpreter.cpp +++ b/lldb/source/Interpreter/CommandInterpreter.cpp @@ -759,6 +759,24 @@ void CommandInterpreter::LoadCommandDictionary() { @@ -4290,7 +4329,7 @@ index 00e9ccb76..2137a1ac8 100644 new CommandObjectRegexCommand( *this, "kdp-remote", diff --git a/lldb/source/Plugins/CMakeLists.txt b/lldb/source/Plugins/CMakeLists.txt -index 9181a4e47..2be6ec365 100644 +index 9181a4e47675..2be6ec3657c0 100644 --- a/lldb/source/Plugins/CMakeLists.txt +++ b/lldb/source/Plugins/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(ABI) @@ -4320,14 +4359,14 @@ index 9181a4e47..2be6ec365 100644 endif() diff --git a/lldb/source/Plugins/DWARFEvaluator/CMakeLists.txt b/lldb/source/Plugins/DWARFEvaluator/CMakeLists.txt new file mode 100644 -index 000000000..73fad41e1 +index 000000000000..73fad41e1a72 --- /dev/null +++ b/lldb/source/Plugins/DWARFEvaluator/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(wasm) diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/CMakeLists.txt b/lldb/source/Plugins/DWARFEvaluator/wasm/CMakeLists.txt new file mode 100644 -index 000000000..e50b1bef7 +index 000000000000..e50b1bef7e69 --- /dev/null +++ b/lldb/source/Plugins/DWARFEvaluator/wasm/CMakeLists.txt @@ -0,0 +1,10 @@ @@ -4343,7 +4382,7 @@ index 000000000..e50b1bef7 + ) diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.cpp b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.cpp new file mode 100644 -index 000000000..fdda1991d +index 000000000000..fdda1991d19f --- /dev/null +++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.cpp @@ -0,0 +1,126 @@ @@ -4475,7 +4514,7 @@ index 000000000..fdda1991d +} diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.h b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.h new file mode 100644 -index 000000000..a01159064 +index 000000000000..a01159064a39 --- /dev/null +++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluator.h @@ -0,0 +1,47 @@ @@ -4528,7 +4567,7 @@ index 000000000..a01159064 +#endif // LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATOR_H diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.cpp b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.cpp new file mode 100644 -index 000000000..d43e96a34 +index 000000000000..d43e96a34d37 --- /dev/null +++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.cpp @@ -0,0 +1,64 @@ @@ -4598,7 +4637,7 @@ index 000000000..d43e96a34 +} diff --git a/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.h b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.h new file mode 100644 -index 000000000..8a946592a +index 000000000000..8a946592a09a --- /dev/null +++ b/lldb/source/Plugins/DWARFEvaluator/wasm/WasmDWARFEvaluatorFactory.h @@ -0,0 +1,55 @@ @@ -4658,7 +4697,7 @@ index 000000000..8a946592a + +#endif // LLDB_SOURCE_PLUGINS_DWARFEVALUATOR_WASM_WASMDWARFEVALUATORFACTORY_H diff --git a/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp b/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp -index ae7e011ea..24ea75d19 100644 +index ae7e011eaa52..24ea75d1971c 100644 --- a/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp +++ b/lldb/source/Plugins/DynamicLoader/wasm-DYLD/DynamicLoaderWasmDYLD.cpp @@ -62,6 +62,15 @@ void DynamicLoaderWasmDYLD::DidAttach() { @@ -4678,7 +4717,7 @@ index ae7e011ea..24ea75d19 100644 ThreadPlanSP DynamicLoaderWasmDYLD::GetStepThroughTrampolinePlan(Thread &thread, diff --git a/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp b/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp -index 5272da9ab..abc5523bf 100644 +index 5272da9ab33a..abc5523bfd70 100644 --- a/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp +++ b/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp @@ -23,6 +23,7 @@ @@ -4718,7 +4757,7 @@ index 5272da9ab..abc5523bf 100644 } } diff --git a/lldb/source/Plugins/Platform/CMakeLists.txt b/lldb/source/Plugins/Platform/CMakeLists.txt -index 5f284e517..6084cbc93 100644 +index 5f284e517dca..6084cbc9378d 100644 --- a/lldb/source/Plugins/Platform/CMakeLists.txt +++ b/lldb/source/Plugins/Platform/CMakeLists.txt @@ -15,3 +15,4 @@ @@ -4728,7 +4767,7 @@ index 5f284e517..6084cbc93 100644 +add_subdirectory(wasm-remote) diff --git a/lldb/source/Plugins/Platform/wasm-remote/CMakeLists.txt b/lldb/source/Plugins/Platform/wasm-remote/CMakeLists.txt new file mode 100644 -index 000000000..4a65765a5 +index 000000000000..4a65765a5659 --- /dev/null +++ b/lldb/source/Plugins/Platform/wasm-remote/CMakeLists.txt @@ -0,0 +1,10 @@ @@ -4744,7 +4783,7 @@ index 000000000..4a65765a5 + ) diff --git a/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.cpp b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.cpp new file mode 100644 -index 000000000..f26d11f00 +index 000000000000..f26d11f00e5c --- /dev/null +++ b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.cpp @@ -0,0 +1,139 @@ @@ -4890,7 +4929,7 @@ index 000000000..f26d11f00 \ No newline at end of file diff --git a/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.h b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.h new file mode 100644 -index 000000000..f306a79d3 +index 000000000000..f306a79d3f4f --- /dev/null +++ b/lldb/source/Plugins/Platform/wasm-remote/PlatformRemoteWasmServer.h @@ -0,0 +1,37 @@ @@ -4933,7 +4972,7 @@ index 000000000..f306a79d3 +#endif \ No newline at end of file diff --git a/lldb/source/Plugins/Plugins.def.in b/lldb/source/Plugins/Plugins.def.in -index bf54598fb..b0bd7b996 100644 +index bf54598fb2f3..b0bd7b9965fe 100644 --- a/lldb/source/Plugins/Plugins.def.in +++ b/lldb/source/Plugins/Plugins.def.in @@ -31,6 +31,7 @@ @@ -4945,7 +4984,7 @@ index bf54598fb..b0bd7b996 100644 #undef LLDB_PLUGIN diff --git a/lldb/source/Plugins/Process/CMakeLists.txt b/lldb/source/Plugins/Process/CMakeLists.txt -index bea5bac9e..7a0855e02 100644 +index bea5bac9eb21..7a0855e02ca2 100644 --- a/lldb/source/Plugins/Process/CMakeLists.txt +++ b/lldb/source/Plugins/Process/CMakeLists.txt @@ -18,3 +18,4 @@ add_subdirectory(Utility) @@ -4954,7 +4993,7 @@ index bea5bac9e..7a0855e02 100644 add_subdirectory(minidump) +add_subdirectory(wasm) diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp -index 12bc7390c..707ab85e5 100644 +index 12bc7390c729..707ab85e5615 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp @@ -285,7 +285,7 @@ bool ProcessElfCore::IsAlive() { return true; } @@ -4967,7 +5006,7 @@ index 12bc7390c..707ab85e5 100644 // in core files we have it all cached our our core file anyway. return DoReadMemory(addr, buf, size, error); diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h -index d8e3cc9ae..f0bf9c4d3 100644 +index d8e3cc9ae3e1..f0bf9c4d3b00 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.h @@ -84,7 +84,8 @@ public: @@ -4981,7 +5020,7 @@ index d8e3cc9ae..f0bf9c4d3 100644 size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, lldb_private::Status &error) override; diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp -index 6914b3734..bb8a05604 100644 +index 6914b37348ea..bb8a056049f3 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -334,6 +334,11 @@ ConstString ProcessGDBRemote::GetPluginName() { return GetPluginNameStatic(); } @@ -5015,7 +5054,7 @@ index 6914b3734..bb8a05604 100644 } } diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h -index fe04cdddd..e4a14c645 100644 +index fe04cdddd0f5..e4a14c64579a 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h @@ -237,6 +237,8 @@ protected: @@ -5028,7 +5067,7 @@ index fe04cdddd..e4a14c645 100644 enum { eBroadcastBitAsyncContinue = (1 << 0), diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp -index 84548edb5..0ae6f7e4a 100644 +index 84548edb5caa..0ae6f7e4a177 100644 --- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp +++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp @@ -596,7 +596,7 @@ bool ProcessMachCore::WarnBeforeDetach() const { return false; } @@ -5041,7 +5080,7 @@ index 84548edb5..0ae6f7e4a 100644 // in core files we have it all cached our our core file anyway. return DoReadMemory(addr, buf, size, error); diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h -index db77e96f1..1c930896c 100644 +index db77e96f1072..1c930896c743 100644 --- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h +++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.h @@ -65,7 +65,8 @@ public: @@ -5055,7 +5094,7 @@ index db77e96f1..1c930896c 100644 size_t DoReadMemory(lldb::addr_t addr, void *buf, size_t size, lldb_private::Status &error) override; diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp -index 385557422..d8bb21581 100644 +index 385557422758..d8bb21581086 100644 --- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp +++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp @@ -374,7 +374,7 @@ bool ProcessMinidump::IsAlive() { return true; } @@ -5068,7 +5107,7 @@ index 385557422..d8bb21581 100644 // we have it all cached in our dump file anyway. return DoReadMemory(addr, buf, size, error); diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.h b/lldb/source/Plugins/Process/minidump/ProcessMinidump.h -index 27b0da004..e94ecab43 100644 +index 27b0da0047a5..e94ecab430c1 100644 --- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.h +++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.h @@ -69,8 +69,8 @@ public: @@ -5084,7 +5123,7 @@ index 27b0da004..e94ecab43 100644 Status &error) override; diff --git a/lldb/source/Plugins/Process/wasm/CMakeLists.txt b/lldb/source/Plugins/Process/wasm/CMakeLists.txt new file mode 100644 -index 000000000..61efb933f +index 000000000000..61efb933fa62 --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/CMakeLists.txt @@ -0,0 +1,12 @@ @@ -5102,7 +5141,7 @@ index 000000000..61efb933f + ) diff --git a/lldb/source/Plugins/Process/wasm/ProcessWasm.cpp b/lldb/source/Plugins/Process/wasm/ProcessWasm.cpp new file mode 100644 -index 000000000..9c0fc7b7f +index 000000000000..9c0fc7b7f270 --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/ProcessWasm.cpp @@ -0,0 +1,261 @@ @@ -5369,7 +5408,7 @@ index 000000000..9c0fc7b7f +} diff --git a/lldb/source/Plugins/Process/wasm/ProcessWasm.h b/lldb/source/Plugins/Process/wasm/ProcessWasm.h new file mode 100644 -index 000000000..d3aece7a6 +index 000000000000..d3aece7a6554 --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/ProcessWasm.h @@ -0,0 +1,128 @@ @@ -5503,7 +5542,7 @@ index 000000000..d3aece7a6 +#endif // LLDB_SOURCE_PLUGINS_PROCESS_WASM_PROCESSWASM_H diff --git a/lldb/source/Plugins/Process/wasm/ThreadWasm.cpp b/lldb/source/Plugins/Process/wasm/ThreadWasm.cpp new file mode 100644 -index 000000000..fa02073e7 +index 000000000000..fa02073e7a52 --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/ThreadWasm.cpp @@ -0,0 +1,35 @@ @@ -5544,7 +5583,7 @@ index 000000000..fa02073e7 +} diff --git a/lldb/source/Plugins/Process/wasm/ThreadWasm.h b/lldb/source/Plugins/Process/wasm/ThreadWasm.h new file mode 100644 -index 000000000..0a33c07de +index 000000000000..0a33c07de994 --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/ThreadWasm.h @@ -0,0 +1,41 @@ @@ -5591,7 +5630,7 @@ index 000000000..0a33c07de +#endif // LLDB_SOURCE_PLUGINS_PROCESS_WASM_THREADWASM_H diff --git a/lldb/source/Plugins/Process/wasm/UnwindWasm.cpp b/lldb/source/Plugins/Process/wasm/UnwindWasm.cpp new file mode 100644 -index 000000000..1a195cb93 +index 000000000000..1a195cb9361a --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/UnwindWasm.cpp @@ -0,0 +1,74 @@ @@ -5672,7 +5711,7 @@ index 000000000..1a195cb93 \ No newline at end of file diff --git a/lldb/source/Plugins/Process/wasm/UnwindWasm.h b/lldb/source/Plugins/Process/wasm/UnwindWasm.h new file mode 100644 -index 000000000..9bd1dac9a +index 000000000000..9bd1dac9a98a --- /dev/null +++ b/lldb/source/Plugins/Process/wasm/UnwindWasm.h @@ -0,0 +1,55 @@ @@ -5732,7 +5771,7 @@ index 000000000..9bd1dac9a + +#endif // lldb_UnwindWasm_h_ diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp -index ccaf31317..c3ef5aebd 100644 +index ccaf31317d75..c3ef5aebd46d 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -3212,8 +3212,13 @@ VariableSP SymbolFileDWARF::ParseVariableDIE(const SymbolContext &sc, @@ -5751,7 +5790,7 @@ index ccaf31317..c3ef5aebd 100644 // DWARF doesn't specify if a DW_TAG_variable is a local, global // or static variable, so we have to do a little digging: diff --git a/lldb/source/Target/PathMappingList.cpp b/lldb/source/Target/PathMappingList.cpp -index b660c310e..cd76421ce 100644 +index b660c310ef31..cd76421cec18 100644 --- a/lldb/source/Target/PathMappingList.cpp +++ b/lldb/source/Target/PathMappingList.cpp @@ -218,7 +218,12 @@ bool PathMappingList::ReverseRemapPath(const FileSpec &file, FileSpec &fixed) co @@ -5769,7 +5808,7 @@ index b660c310e..cd76421ce 100644 return {}; diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp -index a77ecddfb..e257f9350 100644 +index a77ecddfbab6..e257f93508f6 100644 --- a/lldb/source/Target/Platform.cpp +++ b/lldb/source/Target/Platform.cpp @@ -1970,6 +1970,12 @@ size_t Platform::GetSoftwareBreakpointTrapOpcode(Target &target, @@ -5786,7 +5825,7 @@ index a77ecddfb..e257f9350 100644 return 0; } diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp -index 8ecc66b59..f14898791 100644 +index 8ecc66b592ea..f148987915de 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -1892,7 +1892,8 @@ Status Process::DisableSoftwareBreakpoint(BreakpointSite *bp_site) { @@ -5800,7 +5839,7 @@ index 8ecc66b59..f14898791 100644 if (!GetDisableMemoryCache()) { #if defined(VERIFY_MEMORY_READS) diff --git a/lldb/source/Target/ProcessTrace.cpp b/lldb/source/Target/ProcessTrace.cpp -index c878a2ac4..ad5945b0a 100644 +index c878a2ac4eb9..ad5945b0ad1f 100644 --- a/lldb/source/Target/ProcessTrace.cpp +++ b/lldb/source/Target/ProcessTrace.cpp @@ -88,7 +88,7 @@ void ProcessTrace::RefreshStateAfterStop() {} @@ -5813,7 +5852,7 @@ index c878a2ac4..ad5945b0a 100644 // we have it all cached in the trace files. return DoReadMemory(addr, buf, size, error); diff --git a/lldb/source/Target/ThreadPlanStepRange.cpp b/lldb/source/Target/ThreadPlanStepRange.cpp -index 896e647bb..f76307016 100644 +index 896e647bbb52..f76307016102 100644 --- a/lldb/source/Target/ThreadPlanStepRange.cpp +++ b/lldb/source/Target/ThreadPlanStepRange.cpp @@ -334,7 +334,10 @@ bool ThreadPlanStepRange::SetNextBranchBreakpoint() { @@ -5829,7 +5868,7 @@ index 896e647bb..f76307016 100644 instructions->GetInstructionAtIndex(last_index); size_t last_inst_size = last_inst->GetOpcode().GetByteSize(); diff --git a/lldb/source/Target/UnixSignals.cpp b/lldb/source/Target/UnixSignals.cpp -index 4ec2e25c7..24c88fe9a 100644 +index 4ec2e25c7e3b..24c88fe9ae4f 100644 --- a/lldb/source/Target/UnixSignals.cpp +++ b/lldb/source/Target/UnixSignals.cpp @@ -46,6 +46,8 @@ lldb::UnixSignalsSP UnixSignals::Create(const ArchSpec &arch) { @@ -5842,7 +5881,7 @@ index 4ec2e25c7..24c88fe9a 100644 return std::make_shared(); } diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h -index 4310ba9ce..297b33879 100644 +index 4310ba9ce9e0..297b3387999d 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCExecutorProcessControl.h @@ -13,6 +13,7 @@ @@ -5854,7 +5893,7 @@ index 4310ba9ce..297b33879 100644 #include "llvm/ExecutionEngine/Orc/Shared/RPCUtils.h" #include "llvm/ExecutionEngine/Orc/Shared/RawByteChannel.h" diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h -index 753b1998c..27370c62d 100644 +index 753b1998c40c..27370c62dd6e 100644 --- a/llvm/include/llvm/Support/MathExtras.h +++ b/llvm/include/llvm/Support/MathExtras.h @@ -16,6 +16,7 @@ diff --git a/doc/source_debugging.md b/doc/source_debugging.md index a9fa0930..513185fe 100644 --- a/doc/source_debugging.md +++ b/doc/source_debugging.md @@ -44,7 +44,7 @@ iwasm -g=127.0.0.1:1234 test.wasm ``` bash git clone --branch release/13.x --depth=1 https://github.com/llvm/llvm-project cd llvm-project -git apply ${WAMR_ROOT}/build-scripts/lldb-wasm.patch +git apply ${WAMR_ROOT}/build-scripts/lldb_wasm.patch mkdir build-lldb cmake -S ./llvm -B build-lldb \ -G Ninja \