From 5e20cf383e90cb63750fc90408791c7ffba6fffa Mon Sep 17 00:00:00 2001 From: Wenyong Huang Date: Wed, 18 Sep 2024 14:53:41 +0800 Subject: [PATCH] Refactor shared heap feature for interpreter mode (#3794) To add test cases and samples. --- core/iwasm/common/wasm_memory.c | 490 +++++++++--------- core/iwasm/common/wasm_shared_memory.c | 42 +- core/iwasm/interpreter/wasm_interp_classic.c | 215 +++++--- core/iwasm/interpreter/wasm_interp_fast.c | 168 ++++-- core/iwasm/interpreter/wasm_runtime.h | 4 +- .../libraries/thread-mgr/thread_manager.c | 9 + 6 files changed, 543 insertions(+), 385 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index dc9c4aa1..48c87e76 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -218,7 +218,6 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf, fail4: wasm_munmap_linear_memory(heap->base_addr, size, size); - fail3: wasm_runtime_free(heap->heap_handle); fail2: @@ -227,6 +226,45 @@ fail1: return NULL; } +bool +wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, + WASMSharedHeap *shared_heap) +{ + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + uint64 linear_mem_size; + + if (!memory) + return false; + + linear_mem_size = memory->memory_data_size; + + /* check if linear memory and shared heap are overlapped */ + if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64) + || (!memory->is_memory64 + && linear_mem_size > shared_heap->start_off_mem32)) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + return false; + } + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + if (((WASMModuleInstance *)module_inst)->e->shared_heap) { + LOG_WARNING("A shared heap is already attached"); + return false; + } + ((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +#endif + + return true; +} + bool wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, WASMSharedHeap *shared_heap) @@ -238,42 +276,19 @@ wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, #endif } -bool -wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, - WASMSharedHeap *shared_heap) +void +wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) { - uint64 linear_mem_size = 0; - WASMMemoryInstance *memory = NULL; - WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap; - +#if WASM_ENABLE_INTERP != 0 if (module_inst->module_type == Wasm_Module_Bytecode) { - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); + ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; } - else if (module_inst->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { // TODO } - - // check if linear memory and shared heap are overlapped - linear_mem_size = memory->memory_data_size; - - if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64) - || (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - return false; - } - - if (module_inst->module_type == Wasm_Module_Bytecode) { - if (((WASMModuleInstance *)module_inst)->e->shared_heap) { - LOG_WARNING("A shared heap is already attached"); - return false; - } - ((WASMModuleInstance *)module_inst)->e->shared_heap = heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - return true; +#endif } void @@ -286,28 +301,28 @@ wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst) #endif } -void -wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) +static WASMSharedHeap * +get_shared_heap(WASMModuleInstanceCommon *module_inst_comm) { - if (module_inst->module_type == Wasm_Module_Bytecode) { - ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; +#if WASM_ENABLE_INTERP != 0 + if (module_inst_comm->module_type == Wasm_Module_Bytecode) { + return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; } - else if (module_inst->module_type == Wasm_Module_AoT) { +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst_comm->module_type == Wasm_Module_AoT) { // TODO + return NULL; } +#endif + return NULL; } static bool -is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, +is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, bool is_memory64, uint64 app_offset, uint32 bytes) { - WASMSharedHeap *heap = NULL; - if (module_inst_comm->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; - } - else if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - } + WASMSharedHeap *heap = get_shared_heap(module_inst); if (!heap) { return false; @@ -325,21 +340,15 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, return true; } } + return false; } static bool -is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, uint8 *addr, uint32 bytes) { - WASMSharedHeap *heap = NULL; - - if (module_inst_comm->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap; - } - else if (module_inst_comm->module_type == Wasm_Module_AoT) { - // TODO - } + WASMSharedHeap *heap = get_shared_heap(module_inst); if (heap && addr >= heap->base_addr && addr + bytes <= heap->base_addr + heap->size @@ -349,156 +358,72 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, return false; } -static uint64 -shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst, - WASMMemoryInstance *memory, void *addr) -{ - WASMSharedHeap *heap = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - return 0; - } - if (!addr) { - LOG_WARNING("Invalid address"); - return 0; - } - - if (memory && memory->is_memory64) { - return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr); - } - else if (memory && !memory->is_memory64) { - return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr); - } - return 0; -} - -static void * -shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst, - WASMMemoryInstance *memory, uint64 ptr) -{ - void *addr = NULL; - WASMSharedHeap *heap = NULL; - - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - return NULL; - } - - if (!memory) { - LOG_WARNING("Wasm memory is not initialized"); - return NULL; - } - - if (memory->is_memory64) { - addr = heap->base_addr + (ptr - heap->start_off_mem64); - } - else { - addr = heap->base_addr + (ptr - heap->start_off_mem32); - } - - return addr; -} - -static uint64 -shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory) -{ - uint64 shared_heap_start = 0; - - if (!heap || !memory) { - LOG_ERROR("Invalid heap or memory"); - return 0; - } - - if (memory && !memory->is_memory64) { - shared_heap_start = heap->start_off_mem32; - } - else if (memory && memory->is_memory64) { - shared_heap_start = heap->start_off_mem64; - } - - return shared_heap_start; -} - uint64 wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, uint64_t size, void **p_native_addr) { - WASMSharedHeap *heap = NULL; - WASMMemoryInstance *memory = NULL; + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } + if (!memory || !shared_heap) + return 0; - if (heap) { - *p_native_addr = mem_allocator_malloc(heap->heap_handle, size); + *p_native_addr = mem_allocator_malloc(shared_heap->heap_handle, size); + if (!*p_native_addr) + return 0; - return shared_heap_addr_native_to_app(module_inst, memory, - *p_native_addr); - } - else { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); - } - return 0; + if (memory->is_memory64) + return shared_heap->start_off_mem64 + + ((uint8 *)*p_native_addr - shared_heap->base_addr); + else + return shared_heap->start_off_mem32 + + ((uint8 *)*p_native_addr - shared_heap->base_addr); } void -wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) +wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr) { - WASMSharedHeap *heap = NULL; - WASMMemoryInstance *memory = NULL; - void *addr = NULL; + WASMMemoryInstance *memory = + wasm_get_default_memory((WASMModuleInstance *)module_inst); + WASMSharedHeap *shared_heap = get_shared_heap(module_inst); + uint8 *addr = NULL; - if (module_inst->module_type == Wasm_Module_Bytecode) { - heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; - memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); - } - else if (module_inst->module_type == Wasm_Module_AoT) { - // TODO - } - - if (!heap) { - LOG_WARNING("Wasm module doesn't attach to a shared heap"); + if (!memory || !shared_heap) { return; } - addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); - - if (heap) { - mem_allocator_free(heap->base_addr, addr); + if (memory->is_memory64) { + if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */ + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64); } + else { + if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) { + LOG_WARNING("The address to free isn't in shared heap"); + return; + } + addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32); + } + + mem_allocator_free(shared_heap->heap_handle, addr); } -#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ bool wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, const MemAllocOption *alloc_option) { bool ret = false; + #if WASM_ENABLE_SHARED_HEAP != 0 if (os_mutex_init(&shared_heap_list_lock)) { return false; } #endif + if (mem_alloc_type == Alloc_With_Pool) { ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, alloc_option->pool.heap_size); @@ -516,6 +441,10 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; ret = true; } + else { + ret = false; + } + #if WASM_ENABLE_SHARED_HEAP != 0 if (!ret) { os_mutex_destroy(&shared_heap_list_lock); @@ -527,33 +456,27 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, #if WASM_ENABLE_SHARED_HEAP != 0 static void -wasm_runtime_shared_heap_destroy() +wasm_runtime_destroy_shared_heaps() { WASMSharedHeap *heap = shared_heap_list; WASMSharedHeap *cur; - int ret = 0; while (heap) { cur = heap; heap = heap->next; - ret = ret + mem_allocator_destroy(cur->heap_handle); + mem_allocator_destroy(cur->heap_handle); wasm_runtime_free(cur->heap_handle); wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); wasm_runtime_free(cur); } - - if (ret != 0) { - LOG_ERROR("Memory leak detected in shared heap"); - } } #endif void wasm_runtime_memory_destroy(void) { - #if WASM_ENABLE_SHARED_HEAP != 0 - wasm_runtime_shared_heap_destroy(); + wasm_runtime_destroy_shared_heaps(); #endif if (memory_mode == MEMORY_MODE_POOL) { @@ -730,6 +653,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, size)) { + return true; + } +#endif + #if WASM_ENABLE_MEMORY64 != 0 if (memory_inst->is_memory64) max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; @@ -737,7 +667,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, /* boundary overflow check */ if (size > max_linear_memory_size || app_offset > max_linear_memory_size - size) { - goto shared_heap_bound_check; + goto fail; } SHARED_MEMORY_LOCK(memory_inst); @@ -749,13 +679,6 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); -shared_heap_bound_check: -#if WASM_ENABLE_SHARED_HEAP != 0 - if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, - app_offset, size)) { - return true; - } -#endif fail: wasm_set_exception(module_inst, "out of bounds memory access"); return false; @@ -766,6 +689,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, uint64 app_str_offset) { WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; + WASMMemoryInstance *memory_inst; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; char *str, *str_end; @@ -776,22 +700,42 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } - if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, - &app_end_offset)) + memory_inst = wasm_get_default_memory(module_inst); + if (!memory_inst) { goto fail; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_str_offset, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + str = (char *)shared_heap->base_addr + + (memory_inst->is_memory64 + ? (app_str_offset - shared_heap->start_off_mem64) + : (app_str_offset - shared_heap->start_off_mem32)); + str_end = (char *)shared_heap->base_addr + shared_heap->size; + } + else +#endif + { + if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, + NULL, &app_end_offset)) + goto fail; #if WASM_ENABLE_MEMORY64 != 0 - if (module_inst->memories[0]->is_memory64) - max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; + if (memory_inst->is_memory64) + max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; #endif - /* boundary overflow check, max start offset can only be size - 1, while end - * offset can be size */ - if (app_str_offset >= max_linear_memory_size - || app_end_offset > max_linear_memory_size) - goto fail; + /* boundary overflow check, max start offset can be size - 1, while end + offset can be size */ + if (app_str_offset >= max_linear_memory_size + || app_end_offset > max_linear_memory_size) + goto fail; + + str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); + str_end = str + (app_end_offset - app_str_offset); + } - str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset); - str_end = str + (app_end_offset - app_str_offset); while (str < str_end && *str != '\0') str++; if (str == str_end) @@ -833,6 +777,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) { + return true; + } +#endif + SHARED_MEMORY_LOCK(memory_inst); if (memory_inst->memory_data <= addr @@ -841,13 +791,6 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm, return true; } -#if WASM_ENABLE_SHARED_HEAP != 0 - else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, - size)) { - SHARED_MEMORY_UNLOCK(memory_inst); - return true; - } -#endif SHARED_MEMORY_UNLOCK(memory_inst); fail: @@ -874,6 +817,23 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, return NULL; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, + app_offset, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + uint64 shared_heap_start = 0; + + if (memory_inst && !memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem32; + } + else if (memory_inst && memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem64; + } + + return shared_heap->base_addr + app_offset - shared_heap_start; + } +#endif + SHARED_MEMORY_LOCK(memory_inst); addr = memory_inst->memory_data + (uintptr_t)app_offset; @@ -884,19 +844,6 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return addr; } -#if WASM_ENABLE_SHARED_HEAP != 0 - else if (is_app_addr_in_shared_heap(module_inst_comm, - memory_inst->is_memory64, - app_offset, 1)) { - uint64 heap_start = shared_heap_get_addr_start( - module_inst->e->shared_heap, memory_inst); - uint64 heap_offset = (uint64)app_offset - heap_start; - - addr = module_inst->e->shared_heap->base_addr + heap_offset; - SHARED_MEMORY_UNLOCK(memory_inst); - return addr; - } -#endif SHARED_MEMORY_UNLOCK(memory_inst); return NULL; } @@ -931,6 +878,22 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, return 0; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) { + WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm); + uint64 shared_heap_start = 0; + + if (memory_inst && !memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem32; + } + else if (memory_inst && memory_inst->is_memory64) { + shared_heap_start = shared_heap->start_off_mem64; + } + + return shared_heap_start + (addr - shared_heap->base_addr); + } +#endif + SHARED_MEMORY_LOCK(memory_inst); if (bounds_checks) { @@ -940,17 +903,6 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, SHARED_MEMORY_UNLOCK(memory_inst); return ret; } - else { -#if WASM_ENABLE_SHARED_HEAP != 0 - uint64 shared_heap_start = shared_heap_get_addr_start( - module_inst->e->shared_heap, memory_inst); - ret = - (uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr) - + shared_heap_start; - SHARED_MEMORY_UNLOCK(memory_inst); - return ret; -#endif - } } /* If bounds checks is disabled, return the offset directly */ else if (addr != NULL) { @@ -1039,6 +991,10 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); uint8 *native_addr; bool bounds_checks; +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap; + bool is_in_shared_heap = false; +#endif bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); @@ -1047,9 +1003,25 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, return false; } - native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + memory_inst->is_memory64, app_buf_addr, + app_buf_size)) { + shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst); + native_addr = shared_heap->base_addr + + (memory_inst->is_memory64 + ? (app_buf_addr - shared_heap->start_off_mem64) + : (app_buf_addr - shared_heap->start_off_mem32)); + is_in_shared_heap = true; + } + else +#endif + { + native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; + } - bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst); + bounds_checks = + is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst); if (!bounds_checks) { if (app_buf_addr == 0) { @@ -1058,6 +1030,24 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, goto success; } +#if WASM_ENABLE_SHARED_HEAP != 0 + if (is_in_shared_heap) { + const char *str, *str_end; + + /* The whole string must be in the linear memory */ + str = (const char *)native_addr; + str_end = (const char *)shared_heap->base_addr + shared_heap->size; + while (str < str_end && *str != '\0') + str++; + if (str == str_end) { + wasm_set_exception(module_inst, "out of bounds memory access"); + return false; + } + else + goto success; + } +#endif + /* No need to check the app_offset and buf_size if memory access boundary check with hardware trap is enabled */ #ifndef OS_ENABLE_HW_BOUND_CHECK @@ -1205,7 +1195,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, #endif #if WASM_ENABLE_SHARED_HEAP != 0 - WASMSharedHeap *heap; + WASMSharedHeap *shared_heap; #endif uint8 *memory_data_old, *memory_data_new, *heap_data_old; @@ -1241,16 +1231,20 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count, total_size_new = num_bytes_per_page * (uint64)total_page_count; #if WASM_ENABLE_SHARED_HEAP != 0 - heap = module->e->shared_heap; - if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - ret = false; - goto return_func; - } - else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { - LOG_WARNING("Linear memory address is overlapped with shared heap"); - ret = false; - goto return_func; + shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module); + if (shared_heap) { + if (memory->is_memory64 + && total_size_new > shared_heap->start_off_mem64) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } + else if (!memory->is_memory64 + && total_size_new > shared_heap->start_off_mem32) { + LOG_WARNING("Linear memory address is overlapped with shared heap"); + ret = false; + goto return_func; + } } #endif if (inc_page_count <= 0) diff --git a/core/iwasm/common/wasm_shared_memory.c b/core/iwasm/common/wasm_shared_memory.c index 9cfdd092..e0d86d6f 100644 --- a/core/iwasm/common/wasm_shared_memory.c +++ b/core/iwasm/common/wasm_shared_memory.c @@ -243,6 +243,29 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info, destroy_wait_info(wait_info); } +#if WASM_ENABLE_SHARED_HEAP != 0 +static bool +is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, + uint8 *addr, uint32 bytes) +{ + WASMSharedHeap *shared_heap = NULL; + +#if WASM_ENABLE_INTERP != 0 + if (module_inst->module_type == Wasm_Module_Bytecode) { + shared_heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; + } +#endif +#if WASM_ENABLE_AOT != 0 + if (module_inst->module_type == Wasm_Module_AoT) { + // TODO + } +#endif + + return shared_heap && addr >= shared_heap->base_addr + && addr + bytes <= shared_heap->base_addr + shared_heap->size; +} +#endif + uint32 wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, uint64 expect, int64 timeout, bool wait64) @@ -271,9 +294,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, } shared_memory_lock(module_inst->memories[0]); - if ((uint8 *)address < module_inst->memories[0]->memory_data - || (uint8 *)address + (wait64 ? 8 : 4) - > module_inst->memories[0]->memory_data_end) { + if ( +#if WASM_ENABLE_SHARED_HEAP != 0 + /* not in shared heap */ + !is_native_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst, + address, wait64 ? 8 : 4) + && +#endif + /* and not in linear memory */ + ((uint8 *)address < module_inst->memories[0]->memory_data + || (uint8 *)address + (wait64 ? 8 : 4) + > module_inst->memories[0]->memory_data_end)) { shared_memory_unlock(module_inst->memories[0]); wasm_runtime_set_exception(module, "out of bounds memory access"); return -1; @@ -397,6 +428,11 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address, shared_memory_lock(module_inst->memories[0]); out_of_bounds = +#if WASM_ENABLE_SHARED_HEAP != 0 + /* not in shared heap */ + !is_native_addr_in_shared_heap(module, address, 4) && +#endif + /* and not in linear memory */ ((uint8 *)address < module_inst->memories[0]->memory_data || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end); shared_memory_unlock(module_inst->memories[0]); diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c index f39ae346..ecbe408f 100644 --- a/core/iwasm/interpreter/wasm_interp_classic.c +++ b/core/iwasm/interpreter/wasm_interp_classic.c @@ -46,105 +46,89 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if WASM_ENABLE_MEMORY64 == 0 - -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ - || WASM_ENABLE_BULK_MEMORY != 0 -#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; -#else -#define GOTO_OUT_OF_BOUNDS -#endif - #if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ - && offset1 + bytes <= UINT32_MAX) { \ - uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->shared_heap->data + heap_offset; \ - } \ - else { \ - GOTO_OUT_OF_BOUNDS; \ - } \ - } while (0) +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else #else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) #endif +#if WASM_ENABLE_MEMORY64 == 0 + #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* If offset1 is in valid range, maddr must also \ be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* App heap space is not valid space for \ bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ - } while (0) -#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ - WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - maddr = memory->memory_data + offset1; \ + goto out_of_bounds; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ - do { \ - maddr = memory->memory_data + (uint32)(start); \ - uint64 offset1 = start; \ +#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ + WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ + +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) + +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ + } while (0) + #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */ -#if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ - && offset1 + bytes <= UINT64_MAX) { \ - uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->shared_heap->data + heap_offset; \ - } \ - else \ - goto out_of_bounds; \ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \ + if (disable_bounds_checks \ + || (offset1 >= offset && offset1 + bytes >= offset1 \ + && offset1 + bytes <= get_linear_mem_size())) \ + maddr = memory->memory_data + offset1; \ + else \ + goto out_of_bounds; \ } while (0) -#else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds; -#endif -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - /* If memory64 is enabled, offset1, offset1 + bytes can \ - * overflow */ \ - if (disable_bounds_checks \ - || (offset1 >= offset && offset1 + bytes >= offset1 \ - && offset1 + bytes <= get_linear_mem_size())) \ - maddr = memory->memory_data + offset1; \ - else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ - } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint64)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ /* If memory64 is enabled, offset1 + bytes can overflow */ \ if (disable_bounds_checks \ || (offset1 + bytes >= offset1 \ @@ -153,7 +137,7 @@ typedef float64 CellType_F64; bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ + goto out_of_bounds; \ } while (0) #endif /* end of WASM_ENABLE_MEMORY64 == 0 */ @@ -1648,6 +1632,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, if (memory) is_memory64 = memory->is_memory64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap = module->e->shared_heap; + uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; +#if WASM_ENABLE_MEMORY64 != 0 + uint64 shared_heap_start_off = + shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32) + : 0; + uint64 shared_heap_end_off = + shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; +#else + uint64 shared_heap_start_off = + shared_heap ? shared_heap->start_off_mem32 : 0; + uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; +#endif +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_MULTI_MEMORY != 0 uint32 memidx = 0; uint32 memidx_cached = (uint32)-1; @@ -3498,8 +3498,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, str_obj = (WASMString)wasm_stringref_obj_get_value( stringref_obj); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } if (opcode == WASM_OP_STRING_ENCODE_WTF16) { flag = WTF16; @@ -3666,8 +3673,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); stringview_wtf8_obj = POP_REF(); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } bytes_written = wasm_string_encode( (WASMString)wasm_stringview_wtf8_obj_get_value( @@ -5694,9 +5708,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); #else - if ((uint64)(uint32)addr + bytes > linear_mem_size) - goto out_of_bounds; - maddr = memory->memory_data + (uint32)addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } #endif if (bh_bitmap_get_bit(module->e->common.data_dropped, @@ -5746,15 +5769,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_THREAD_MGR != 0 linear_mem_size = get_linear_mem_size(); #endif + + dlen = linear_mem_size - dst; + /* dst boundary check */ #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); -#else - if ((uint64)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) + dlen = shared_heap_end_off - dst + 1; #endif - dlen = linear_mem_size - dst; +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + dlen = shared_heap_end_off - dst + 1; + } + else +#endif + { + if ((uint64)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ #if WASM_ENABLE_MULTI_MEMORY != 0 /* src memidx */ @@ -5770,9 +5808,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); #else - if ((uint64)src + len > linear_mem_size) - goto out_of_bounds; - msrc = memory->memory_data + src; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + src; + } #endif #if WASM_ENABLE_MEMORY64 == 0 @@ -5809,9 +5854,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #else - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } #endif memset(mdst, fill_val, len); diff --git a/core/iwasm/interpreter/wasm_interp_fast.c b/core/iwasm/interpreter/wasm_interp_fast.c index 84170f21..82b6ff71 100644 --- a/core/iwasm/interpreter/wasm_interp_fast.c +++ b/core/iwasm/interpreter/wasm_interp_fast.c @@ -37,29 +37,20 @@ typedef float64 CellType_F64; #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #endif -#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ - || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \ - || WASM_ENABLE_BULK_MEMORY != 0 -#define GOTO_OUT_OF_BOUNDS goto out_of_bounds; -#else -#define GOTO_OUT_OF_BOUNDS -#endif - #if WASM_ENABLE_SHARED_HEAP != 0 -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ - do { \ - if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ - && offset1 + bytes <= UINT32_MAX) { \ - uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ - uint64 heap_offset = (uint64)offset1 - heap_start; \ - maddr = module->e->shared_heap->base_addr + heap_offset; \ - } \ - else { \ - GOTO_OUT_OF_BOUNDS; \ - } \ - } while (0) +#define app_addr_in_shared_heap(app_addr, bytes) \ + (shared_heap && (app_addr) >= shared_heap_start_off \ + && (app_addr) <= shared_heap_end_off - bytes + 1) + +#define shared_heap_addr_app_to_native(app_addr, native_addr) \ + native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off) + +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \ + if (app_addr_in_shared_heap(app_addr, bytes)) \ + shared_heap_addr_app_to_native(app_addr, native_addr); \ + else #else -#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS +#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) #endif #if !defined(OS_ENABLE_HW_BOUND_CHECK) \ @@ -67,35 +58,39 @@ typedef float64 CellType_F64; #define CHECK_MEMORY_OVERFLOW(bytes) \ do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* If offset1 is in valid range, maddr must also \ be in valid range, no need to check it again. */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ + goto out_of_bounds; \ } while (0) #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ do { \ uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ /* App heap space is not valid space for \ bulk memory operation */ \ maddr = memory->memory_data + offset1; \ else \ - CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ + goto out_of_bounds; \ } while (0) #else -#define CHECK_MEMORY_OVERFLOW(bytes) \ - do { \ - uint64 offset1 = (uint64)offset + (uint64)addr; \ - maddr = memory->memory_data + offset1; \ +#define CHECK_MEMORY_OVERFLOW(bytes) \ + do { \ + uint64 offset1 = (uint64)offset + (uint64)addr; \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) -#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ - do { \ - maddr = memory->memory_data + (uint32)(start); \ - uint64 offset1 = start; \ +#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ + do { \ + uint64 offset1 = (uint32)(start); \ + CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \ + maddr = memory->memory_data + offset1; \ } while (0) #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ @@ -1542,6 +1537,24 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 bool is_return_call = false; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL; + uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL; + /* +#if WASM_ENABLE_MEMORY64 != 0 + uint64 shared_heap_start_off = + shared_heap ? (is_memory64 ? shared_heap->start_off_mem64 + : shared_heap->start_off_mem32) + : 0; + uint64 shared_heap_end_off = + shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0; +#else + */ /* TODO: uncomment the code when memory64 is enabled for fast-interp */ + uint64 shared_heap_start_off = + shared_heap ? shared_heap->start_off_mem32 : 0; + uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0; +/* #endif */ +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ #if WASM_ENABLE_LABELS_AS_VALUES != 0 #define HANDLE_OPCODE(op) &&HANDLE_##op @@ -2857,8 +2870,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, str_obj = (WASMString)wasm_stringref_obj_get_value( stringref_obj); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } if (opcode == WASM_OP_STRING_ENCODE_WTF16) { flag = WTF16; @@ -3025,8 +3045,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, addr = POP_I32(); stringview_wtf8_obj = POP_REF(); - memory_inst = module->memories[mem_idx]; - maddr = memory_inst->memory_data + addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)addr, 1)) + shared_heap_addr_app_to_native((uint64)addr, maddr); + else +#endif + { + memory_inst = module->memories[mem_idx]; + maddr = memory_inst->memory_data + addr; + } bytes_written = wasm_string_encode( (WASMString)wasm_stringview_wtf8_obj_get_value( @@ -5011,9 +5038,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); #else - if ((uint64)(uint32)addr + bytes > linear_mem_size) - goto out_of_bounds; - maddr = memory->memory_data + (uint32)addr; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)addr, + bytes)) + shared_heap_addr_app_to_native((uint64)(uint32)addr, + maddr); + else +#endif + { + if ((uint64)(uint32)addr + bytes > linear_mem_size) + goto out_of_bounds; + maddr = memory->memory_data + (uint32)addr; + } #endif if (bh_bitmap_get_bit(module->e->common.data_dropped, segment)) { @@ -5046,6 +5082,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, { uint32 dst, src, len; uint8 *mdst, *msrc; + uint64 dlen; len = POP_I32(); src = POP_I32(); @@ -5055,22 +5092,43 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, linear_mem_size = get_linear_mem_size(); #endif + dlen = linear_mem_size - dst; + #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); -#else - if ((uint64)(uint32)src + len > linear_mem_size) - goto out_of_bounds; - msrc = memory->memory_data + (uint32)src; - - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) + dlen = shared_heap_end_off - dst + 1; #endif +#else /* else of OS_ENABLE_HW_BOUND_CHECK */ +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)src, len)) + shared_heap_addr_app_to_native((uint64)src, msrc); + else +#endif + { + if ((uint64)(uint32)src + len > linear_mem_size) + goto out_of_bounds; + msrc = memory->memory_data + (uint32)src; + } + +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)dst, len)) { + shared_heap_addr_app_to_native((uint64)dst, mdst); + dlen = shared_heap_end_off - dst + 1; + } + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } +#endif /* end of OS_ENABLE_HW_BOUND_CHECK */ /* allowing the destination and source to overlap */ - bh_memmove_s(mdst, (uint32)(linear_mem_size - dst), - msrc, len); + bh_memmove_s(mdst, (uint32)dlen, msrc, len); break; } case WASM_OP_MEMORY_FILL: @@ -5089,9 +5147,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, #ifndef OS_ENABLE_HW_BOUND_CHECK CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); #else - if ((uint64)(uint32)dst + len > linear_mem_size) - goto out_of_bounds; - mdst = memory->memory_data + (uint32)dst; +#if WASM_ENABLE_SHARED_HEAP != 0 + if (app_addr_in_shared_heap((uint64)(uint32)dst, len)) + shared_heap_addr_app_to_native((uint64)(uint32)dst, + mdst); + else +#endif + { + if ((uint64)(uint32)dst + len > linear_mem_size) + goto out_of_bounds; + mdst = memory->memory_data + (uint32)dst; + } #endif memset(mdst, fill_val, len); diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 3fa3682c..3c4e5faa 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -96,8 +96,8 @@ typedef union { typedef struct WASMSharedHeap { struct WASMSharedHeap *next; void *heap_handle; - uint8_t *base_addr; - uint32_t size; + uint8 *base_addr; + uint32 size; uint64 start_off_mem64; uint64 start_off_mem32; } WASMSharedHeap; diff --git a/core/iwasm/libraries/thread-mgr/thread_manager.c b/core/iwasm/libraries/thread-mgr/thread_manager.c index 27ff1441..46a1bb32 100644 --- a/core/iwasm/libraries/thread-mgr/thread_manager.c +++ b/core/iwasm/libraries/thread-mgr/thread_manager.c @@ -1449,10 +1449,19 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst, bh_assert(cluster); os_mutex_lock(&cluster->lock); + /* Try attaching shared heap to this module instance first + to ensure that we can attach it to all other instances. */ + if (!wasm_runtime_attach_shared_heap_internal(module_inst, heap)) { + os_mutex_unlock(&cluster->lock); + return false; + } + /* Detach the shared heap so it can be attached again. */ + wasm_runtime_detach_shared_heap_internal(module_inst); traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, heap); os_mutex_unlock(&cluster->lock); } + return true; }