christoph/wamr
1
Fork 0
Code Releases Activity

Refactor shared heap feature for interpreter mode (#3794)

Browse Source 
To add test cases and samples.
This commit is contained in:
Wenyong Huang
2024-09-18 14:53:41 +08:00
committed by GitHub
parent 92852f3719
commit 5e20cf383e
6 changed files with 543 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

490
core/iwasm/common/wasm_memory.c
View File

@ -218,7 +218,6 @@ wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args, char *error_buf,
fail4: fail4:
wasm_munmap_linear_memory(heap->base_addr, size, size); wasm_munmap_linear_memory(heap->base_addr, size, size);
fail3: fail3:
wasm_runtime_free(heap->heap_handle); wasm_runtime_free(heap->heap_handle);
fail2: fail2:
@ -227,6 +226,45 @@ fail1:
return NULL; return NULL;
} }
bool
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
uint64 linear_mem_size;
if (!memory)
return false;
linear_mem_size = memory->memory_data_size;
/* check if linear memory and shared heap are overlapped */
if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64)
|| (!memory->is_memory64
&& linear_mem_size > shared_heap->start_off_mem32)) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
return false;
}
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
if (((WASMModuleInstance *)module_inst)->e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
((WASMModuleInstance *)module_inst)->e->shared_heap = shared_heap;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
#endif
return true;
}
bool bool
wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst, wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap) WASMSharedHeap *shared_heap)
@ -238,42 +276,19 @@ wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
#endif #endif
} }
bool void
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst, wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst)
WASMSharedHeap *shared_heap)
{ {
uint64 linear_mem_size = 0; #if WASM_ENABLE_INTERP != 0
WASMMemoryInstance *memory = NULL;
WASMSharedHeap *heap = (WASMSharedHeap *)shared_heap;
if (module_inst->module_type == Wasm_Module_Bytecode) { if (module_inst->module_type == Wasm_Module_Bytecode) {
memory = wasm_get_default_memory((WASMModuleInstance *)module_inst); ((WASMModuleInstance *)module_inst)->e->shared_heap = NULL;
} }
else if (module_inst->module_type == Wasm_Module_AoT) { #endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
// TODO // TODO
} }
#endif
// check if linear memory and shared heap are overlapped
linear_mem_size = memory->memory_data_size;
if ((memory->is_memory64 && linear_mem_size > heap->start_off_mem64)
|| (!memory->is_memory64 && linear_mem_size > heap->start_off_mem32)) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
return false;
}
if (module_inst->module_type == Wasm_Module_Bytecode) {
if (((WASMModuleInstance *)module_inst)->e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
((WASMModuleInstance *)module_inst)->e->shared_heap = heap;
}
else if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
return true;
} }
void void
@ -286,28 +301,28 @@ wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst)
#endif #endif
} }
void static WASMSharedHeap *
wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst) get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
{ {
if (module_inst->module_type == Wasm_Module_Bytecode) { #if WASM_ENABLE_INTERP != 0
((WASMModuleInstance *)module_inst)->e->shared_heap = NULL; if (module_inst_comm->module_type == Wasm_Module_Bytecode) {
return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap;
} }
else if (module_inst->module_type == Wasm_Module_AoT) { #endif
#if WASM_ENABLE_AOT != 0
if (module_inst_comm->module_type == Wasm_Module_AoT) {
// TODO // TODO
return NULL;
} }
#endif
return NULL;
} }
static bool static bool
is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
bool is_memory64, uint64 app_offset, uint32 bytes) bool is_memory64, uint64 app_offset, uint32 bytes)
{ {
WASMSharedHeap *heap = NULL; WASMSharedHeap *heap = get_shared_heap(module_inst);
if (module_inst_comm->module_type == Wasm_Module_Bytecode) {
heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap;
}
else if (module_inst_comm->module_type == Wasm_Module_AoT) {
// TODO
}
if (!heap) { if (!heap) {
return false; return false;
@ -325,21 +340,15 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm,
return true; return true;
} }
} }
return false; return false;
} }
static bool static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm, is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes) uint8 *addr, uint32 bytes)
{ {
WASMSharedHeap *heap = NULL; WASMSharedHeap *heap = get_shared_heap(module_inst);
if (module_inst_comm->module_type == Wasm_Module_Bytecode) {
heap = ((WASMModuleInstance *)module_inst_comm)->e->shared_heap;
}
else if (module_inst_comm->module_type == Wasm_Module_AoT) {
// TODO
}
if (heap && addr >= heap->base_addr if (heap && addr >= heap->base_addr
&& addr + bytes <= heap->base_addr + heap->size && addr + bytes <= heap->base_addr + heap->size
@ -349,156 +358,72 @@ is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst_comm,
return false; return false;
} }
static uint64
shared_heap_addr_native_to_app(WASMModuleInstanceCommon *module_inst,
WASMMemoryInstance *memory, void *addr)
{
WASMSharedHeap *heap = NULL;
if (module_inst->module_type == Wasm_Module_Bytecode) {
heap = ((WASMModuleInstance *)module_inst)->e->shared_heap;
}
else if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
if (!heap) {
LOG_WARNING("Wasm module doesn't attach to a shared heap");
return 0;
}
if (!addr) {
LOG_WARNING("Invalid address");
return 0;
}
if (memory && memory->is_memory64) {
return heap->start_off_mem64 + ((uint8 *)addr - heap->base_addr);
}
else if (memory && !memory->is_memory64) {
return heap->start_off_mem32 + ((uint8 *)addr - heap->base_addr);
}
return 0;
}
static void *
shared_heap_addr_app_to_native(WASMModuleInstanceCommon *module_inst,
WASMMemoryInstance *memory, uint64 ptr)
{
void *addr = NULL;
WASMSharedHeap *heap = NULL;
if (module_inst->module_type == Wasm_Module_Bytecode) {
heap = ((WASMModuleInstance *)module_inst)->e->shared_heap;
}
else if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
if (!heap) {
LOG_WARNING("Wasm module doesn't attach to a shared heap");
return NULL;
}
if (!memory) {
LOG_WARNING("Wasm memory is not initialized");
return NULL;
}
if (memory->is_memory64) {
addr = heap->base_addr + (ptr - heap->start_off_mem64);
}
else {
addr = heap->base_addr + (ptr - heap->start_off_mem32);
}
return addr;
}
static uint64
shared_heap_get_addr_start(WASMSharedHeap *heap, WASMMemoryInstance *memory)
{
uint64 shared_heap_start = 0;
if (!heap || !memory) {
LOG_ERROR("Invalid heap or memory");
return 0;
}
if (memory && !memory->is_memory64) {
shared_heap_start = heap->start_off_mem32;
}
else if (memory && memory->is_memory64) {
shared_heap_start = heap->start_off_mem64;
}
return shared_heap_start;
}
uint64 uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst, wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64_t size, void **p_native_addr) uint64_t size, void **p_native_addr)
{ {
WASMSharedHeap *heap = NULL; WASMMemoryInstance *memory =
WASMMemoryInstance *memory = NULL; wasm_get_default_memory((WASMModuleInstance *)module_inst);
WASMSharedHeap *shared_heap = get_shared_heap(module_inst);
if (module_inst->module_type == Wasm_Module_Bytecode) { if (!memory || !shared_heap)
heap = ((WASMModuleInstance *)module_inst)->e->shared_heap; return 0;
memory = wasm_get_default_memory((WASMModuleInstance *)module_inst);
}
else if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
if (heap) { *p_native_addr = mem_allocator_malloc(shared_heap->heap_handle, size);
*p_native_addr = mem_allocator_malloc(heap->heap_handle, size); if (!*p_native_addr)
return 0;
return shared_heap_addr_native_to_app(module_inst, memory, if (memory->is_memory64)
*p_native_addr); return shared_heap->start_off_mem64
} + ((uint8 *)*p_native_addr - shared_heap->base_addr);
else { else
LOG_WARNING("Wasm module doesn't attach to a shared heap"); return shared_heap->start_off_mem32
} + ((uint8 *)*p_native_addr - shared_heap->base_addr);
return 0;
} }
void void
wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64 ptr) wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr)
{ {
WASMSharedHeap *heap = NULL; WASMMemoryInstance *memory =
WASMMemoryInstance *memory = NULL; wasm_get_default_memory((WASMModuleInstance *)module_inst);
void *addr = NULL; WASMSharedHeap *shared_heap = get_shared_heap(module_inst);
uint8 *addr = NULL;
if (module_inst->module_type == Wasm_Module_Bytecode) { if (!memory || !shared_heap) {
heap = ((WASMModuleInstance *)module_inst)->e->shared_heap;
memory = wasm_get_default_memory((WASMModuleInstance *)module_inst);
}
else if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
if (!heap) {
LOG_WARNING("Wasm module doesn't attach to a shared heap");
return; return;
} }
addr = shared_heap_addr_app_to_native(module_inst, memory, ptr); if (memory->is_memory64) {
if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */
if (heap) { LOG_WARNING("The address to free isn't in shared heap");
mem_allocator_free(heap->base_addr, addr); return;
}
addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64);
} }
else {
if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) {
LOG_WARNING("The address to free isn't in shared heap");
return;
}
addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32);
}
mem_allocator_free(shared_heap->heap_handle, addr);
} }
#endif #endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
bool bool
wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type, wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
const MemAllocOption *alloc_option) const MemAllocOption *alloc_option)
{ {
bool ret = false; bool ret = false;
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
if (os_mutex_init(&shared_heap_list_lock)) { if (os_mutex_init(&shared_heap_list_lock)) {
return false; return false;
} }
#endif #endif
if (mem_alloc_type == Alloc_With_Pool) { if (mem_alloc_type == Alloc_With_Pool) {
ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf, ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf,
alloc_option->pool.heap_size); alloc_option->pool.heap_size);
@ -516,6 +441,10 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR; memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR;
ret = true; ret = true;
} }
else {
ret = false;
}
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
if (!ret) { if (!ret) {
os_mutex_destroy(&shared_heap_list_lock); os_mutex_destroy(&shared_heap_list_lock);
@ -527,33 +456,27 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
static void static void
wasm_runtime_shared_heap_destroy() wasm_runtime_destroy_shared_heaps()
{ {
WASMSharedHeap *heap = shared_heap_list; WASMSharedHeap *heap = shared_heap_list;
WASMSharedHeap *cur; WASMSharedHeap *cur;
int ret = 0;
while (heap) { while (heap) {
cur = heap; cur = heap;
heap = heap->next; heap = heap->next;
ret = ret + mem_allocator_destroy(cur->heap_handle); mem_allocator_destroy(cur->heap_handle);
wasm_runtime_free(cur->heap_handle); wasm_runtime_free(cur->heap_handle);
wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size); wasm_munmap_linear_memory(cur->base_addr, cur->size, cur->size);
wasm_runtime_free(cur); wasm_runtime_free(cur);
} }
if (ret != 0) {
LOG_ERROR("Memory leak detected in shared heap");
}
} }
#endif #endif
void void
wasm_runtime_memory_destroy(void) wasm_runtime_memory_destroy(void)
{ {
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
wasm_runtime_shared_heap_destroy(); wasm_runtime_destroy_shared_heaps();
#endif #endif
if (memory_mode == MEMORY_MODE_POOL) { if (memory_mode == MEMORY_MODE_POOL) {
@ -730,6 +653,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail; goto fail;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, size)) {
return true;
}
#endif
#if WASM_ENABLE_MEMORY64 != 0 #if WASM_ENABLE_MEMORY64 != 0
if (memory_inst->is_memory64) if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
@ -737,7 +667,7 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
/* boundary overflow check */ /* boundary overflow check */
if (size > max_linear_memory_size if (size > max_linear_memory_size
|| app_offset > max_linear_memory_size - size) { || app_offset > max_linear_memory_size - size) {
goto shared_heap_bound_check; goto fail;
} }
SHARED_MEMORY_LOCK(memory_inst); SHARED_MEMORY_LOCK(memory_inst);
@ -749,13 +679,6 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
SHARED_MEMORY_UNLOCK(memory_inst); SHARED_MEMORY_UNLOCK(memory_inst);
shared_heap_bound_check:
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, size)) {
return true;
}
#endif
fail: fail:
wasm_set_exception(module_inst, "out of bounds memory access"); wasm_set_exception(module_inst, "out of bounds memory access");
return false; return false;
@ -766,6 +689,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
uint64 app_str_offset) uint64 app_str_offset)
{ {
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm; WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
WASMMemoryInstance *memory_inst;
uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE; uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE;
char *str, *str_end; char *str, *str_end;
@ -776,22 +700,42 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
return true; return true;
} }
if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL, memory_inst = wasm_get_default_memory(module_inst);
&app_end_offset)) if (!memory_inst) {
goto fail; goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_str_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
str = (char *)shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_str_offset - shared_heap->start_off_mem64)
: (app_str_offset - shared_heap->start_off_mem32));
str_end = (char *)shared_heap->base_addr + shared_heap->size;
}
else
#endif
{
if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset,
NULL, &app_end_offset))
goto fail;
#if WASM_ENABLE_MEMORY64 != 0 #if WASM_ENABLE_MEMORY64 != 0
if (module_inst->memories[0]->is_memory64) if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE; max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
#endif #endif
/* boundary overflow check, max start offset can only be size - 1, while end /* boundary overflow check, max start offset can be size - 1, while end
* offset can be size */ offset can be size */
if (app_str_offset >= max_linear_memory_size if (app_str_offset >= max_linear_memory_size
|| app_end_offset > max_linear_memory_size) || app_end_offset > max_linear_memory_size)
goto fail; goto fail;
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
str_end = str + (app_end_offset - app_str_offset);
}
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
str_end = str + (app_end_offset - app_str_offset);
while (str < str_end && *str != '\0') while (str < str_end && *str != '\0')
str++; str++;
if (str == str_end) if (str == str_end)
@ -833,6 +777,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail; goto fail;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) {
return true;
}
#endif
SHARED_MEMORY_LOCK(memory_inst); SHARED_MEMORY_LOCK(memory_inst);
if (memory_inst->memory_data <= addr if (memory_inst->memory_data <= addr
@ -841,13 +791,6 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
return true; return true;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
else if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr,
size)) {
SHARED_MEMORY_UNLOCK(memory_inst);
return true;
}
#endif
SHARED_MEMORY_UNLOCK(memory_inst); SHARED_MEMORY_UNLOCK(memory_inst);
fail: fail:
@ -874,6 +817,23 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
return NULL; return NULL;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem32;
}
else if (memory_inst && memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem64;
}
return shared_heap->base_addr + app_offset - shared_heap_start;
}
#endif
SHARED_MEMORY_LOCK(memory_inst); SHARED_MEMORY_LOCK(memory_inst);
addr = memory_inst->memory_data + (uintptr_t)app_offset; addr = memory_inst->memory_data + (uintptr_t)app_offset;
@ -884,19 +844,6 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
SHARED_MEMORY_UNLOCK(memory_inst); SHARED_MEMORY_UNLOCK(memory_inst);
return addr; return addr;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
else if (is_app_addr_in_shared_heap(module_inst_comm,
memory_inst->is_memory64,
app_offset, 1)) {
uint64 heap_start = shared_heap_get_addr_start(
module_inst->e->shared_heap, memory_inst);
uint64 heap_offset = (uint64)app_offset - heap_start;
addr = module_inst->e->shared_heap->base_addr + heap_offset;
SHARED_MEMORY_UNLOCK(memory_inst);
return addr;
}
#endif
SHARED_MEMORY_UNLOCK(memory_inst); SHARED_MEMORY_UNLOCK(memory_inst);
return NULL; return NULL;
} }
@ -931,6 +878,22 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
return 0; return 0;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem32;
}
else if (memory_inst && memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem64;
}
return shared_heap_start + (addr - shared_heap->base_addr);
}
#endif
SHARED_MEMORY_LOCK(memory_inst); SHARED_MEMORY_LOCK(memory_inst);
if (bounds_checks) { if (bounds_checks) {
@ -940,17 +903,6 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
SHARED_MEMORY_UNLOCK(memory_inst); SHARED_MEMORY_UNLOCK(memory_inst);
return ret; return ret;
} }
else {
#if WASM_ENABLE_SHARED_HEAP != 0
uint64 shared_heap_start = shared_heap_get_addr_start(
module_inst->e->shared_heap, memory_inst);
ret =
(uint64)(addr - (uint8 *)module_inst->e->shared_heap->base_addr)
+ shared_heap_start;
SHARED_MEMORY_UNLOCK(memory_inst);
return ret;
#endif
}
} }
/* If bounds checks is disabled, return the offset directly */ /* If bounds checks is disabled, return the offset directly */
else if (addr != NULL) { else if (addr != NULL) {
@ -1039,6 +991,10 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst); WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst);
uint8 *native_addr; uint8 *native_addr;
bool bounds_checks; bool bounds_checks;
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
bool is_in_shared_heap = false;
#endif
bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX); bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX);
@ -1047,9 +1003,25 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
return false; return false;
} }
native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr; #if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
memory_inst->is_memory64, app_buf_addr,
app_buf_size)) {
shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst);
native_addr = shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_buf_addr - shared_heap->start_off_mem64)
: (app_buf_addr - shared_heap->start_off_mem32));
is_in_shared_heap = true;
}
else
#endif
{
native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr;
}
bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst); bounds_checks =
is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst);
if (!bounds_checks) { if (!bounds_checks) {
if (app_buf_addr == 0) { if (app_buf_addr == 0) {
@ -1058,6 +1030,24 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
goto success; goto success;
} }
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_in_shared_heap) {
const char *str, *str_end;
/* The whole string must be in the linear memory */
str = (const char *)native_addr;
str_end = (const char *)shared_heap->base_addr + shared_heap->size;
while (str < str_end && *str != '\0')
str++;
if (str == str_end) {
wasm_set_exception(module_inst, "out of bounds memory access");
return false;
}
else
goto success;
}
#endif
/* No need to check the app_offset and buf_size if memory access /* No need to check the app_offset and buf_size if memory access
boundary check with hardware trap is enabled */ boundary check with hardware trap is enabled */
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
@ -1205,7 +1195,7 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count,
#endif #endif
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *heap; WASMSharedHeap *shared_heap;
#endif #endif
uint8 *memory_data_old, *memory_data_new, *heap_data_old; uint8 *memory_data_old, *memory_data_new, *heap_data_old;
@ -1241,16 +1231,20 @@ wasm_enlarge_memory_internal(WASMModuleInstance *module, uint32 inc_page_count,
total_size_new = num_bytes_per_page * (uint64)total_page_count; total_size_new = num_bytes_per_page * (uint64)total_page_count;
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
heap = module->e->shared_heap; shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module);
if (memory->is_memory64 && total_size_new > heap->start_off_mem64) { if (shared_heap) {
LOG_WARNING("Linear memory address is overlapped with shared heap"); if (memory->is_memory64
ret = false; && total_size_new > shared_heap->start_off_mem64) {
goto return_func; LOG_WARNING("Linear memory address is overlapped with shared heap");
} ret = false;
else if (!memory->is_memory64 && total_size_new > heap->start_off_mem32) { goto return_func;
LOG_WARNING("Linear memory address is overlapped with shared heap"); }
ret = false; else if (!memory->is_memory64
goto return_func; && total_size_new > shared_heap->start_off_mem32) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
ret = false;
goto return_func;
}
} }
#endif #endif
if (inc_page_count <= 0) if (inc_page_count <= 0)

42
core/iwasm/common/wasm_shared_memory.c
View File

@ -243,6 +243,29 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info,
destroy_wait_info(wait_info); destroy_wait_info(wait_info);
} }
#if WASM_ENABLE_SHARED_HEAP != 0
static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes)
{
WASMSharedHeap *shared_heap = NULL;
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
shared_heap = ((WASMModuleInstance *)module_inst)->e->shared_heap;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
// TODO
}
#endif
return shared_heap && addr >= shared_heap->base_addr
&& addr + bytes <= shared_heap->base_addr + shared_heap->size;
}
#endif
uint32 uint32
wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address, wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address,
uint64 expect, int64 timeout, bool wait64) uint64 expect, int64 timeout, bool wait64)
@ -271,9 +294,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address,
} }
shared_memory_lock(module_inst->memories[0]); shared_memory_lock(module_inst->memories[0]);
if ((uint8 *)address < module_inst->memories[0]->memory_data if (
|| (uint8 *)address + (wait64 ? 8 : 4) #if WASM_ENABLE_SHARED_HEAP != 0
> module_inst->memories[0]->memory_data_end) { /* not in shared heap */
!is_native_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
address, wait64 ? 8 : 4)
&&
#endif
/* and not in linear memory */
((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + (wait64 ? 8 : 4)
> module_inst->memories[0]->memory_data_end)) {
shared_memory_unlock(module_inst->memories[0]); shared_memory_unlock(module_inst->memories[0]);
wasm_runtime_set_exception(module, "out of bounds memory access"); wasm_runtime_set_exception(module, "out of bounds memory access");
return -1; return -1;
@ -397,6 +428,11 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address,
shared_memory_lock(module_inst->memories[0]); shared_memory_lock(module_inst->memories[0]);
out_of_bounds = out_of_bounds =
#if WASM_ENABLE_SHARED_HEAP != 0
/* not in shared heap */
!is_native_addr_in_shared_heap(module, address, 4) &&
#endif
/* and not in linear memory */
((uint8 *)address < module_inst->memories[0]->memory_data ((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end); || (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end);
shared_memory_unlock(module_inst->memories[0]); shared_memory_unlock(module_inst->memories[0]);

215
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -46,105 +46,89 @@ typedef float64 CellType_F64;
#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory)
#endif #endif
#if WASM_ENABLE_MEMORY64 == 0
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
#define GOTO_OUT_OF_BOUNDS goto out_of_bounds;
#else
#define GOTO_OUT_OF_BOUNDS
#endif
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ #define app_addr_in_shared_heap(app_addr, bytes) \
do { \ (shared_heap && (app_addr) >= shared_heap_start_off \
if (offset1 + bytes >= UINT32_MAX - module->shared_heap->size \ && (app_addr) <= shared_heap_end_off - bytes + 1)
&& offset1 + bytes <= UINT32_MAX) { \
uint64 heap_start = UINT32_MAX - module->shared_heap->size; \ #define shared_heap_addr_app_to_native(app_addr, native_addr) \
uint64 heap_offset = (uint64)offset1 - heap_start; \ native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off)
maddr = module->shared_heap->data + heap_offset; \
} \ #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \
else { \ if (app_addr_in_shared_heap(app_addr, bytes)) \
GOTO_OUT_OF_BOUNDS; \ shared_heap_addr_app_to_native(app_addr, native_addr); \
} \ else
} while (0)
#else #else
#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr)
#endif #endif
#if WASM_ENABLE_MEMORY64 == 0
#if (!defined(OS_ENABLE_HW_BOUND_CHECK) \ #if (!defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0) || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0)
#define CHECK_MEMORY_OVERFLOW(bytes) \ #define CHECK_MEMORY_OVERFLOW(bytes) \
do { \ do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \ uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* If offset1 is in valid range, maddr must also \ /* If offset1 is in valid range, maddr must also \
be in valid range, no need to check it again. */ \ be in valid range, no need to check it again. */ \
maddr = memory->memory_data + offset1; \ maddr = memory->memory_data + offset1; \
else \ else \
CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ goto out_of_bounds; \
} while (0) } while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \ do { \
uint64 offset1 = (uint32)(start); \ uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* App heap space is not valid space for \ /* App heap space is not valid space for \
bulk memory operation */ \ bulk memory operation */ \
maddr = memory->memory_data + offset1; \ maddr = memory->memory_data + offset1; \
else \ else \
CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ goto out_of_bounds; \
} while (0)
#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
maddr = memory->memory_data + offset1; \
} while (0) } while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ #else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
do { \ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
maddr = memory->memory_data + (uint32)(start); \
uint64 offset1 = start; \ #define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0) } while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0)
#endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \ #endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
#else /* else of WASM_ENABLE_MEMORY64 == 0 */ #else /* else of WASM_ENABLE_MEMORY64 == 0 */
#if WASM_ENABLE_SHARED_HEAP != 0 #define CHECK_MEMORY_OVERFLOW(bytes) \
#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ do { \
do { \ uint64 offset1 = (uint64)offset + (uint64)addr; \
if (offset1 + bytes >= UINT64_MAX - module->shared_heap->size \ CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
&& offset1 + bytes <= UINT64_MAX) { \ /* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \
uint64 heap_start = UINT64_MAX - module->shared_heap->size; \ if (disable_bounds_checks \
uint64 heap_offset = (uint64)offset1 - heap_start; \ || (offset1 >= offset && offset1 + bytes >= offset1 \
maddr = module->shared_heap->data + heap_offset; \ && offset1 + bytes <= get_linear_mem_size())) \
} \ maddr = memory->memory_data + offset1; \
else \ else \
goto out_of_bounds; \ goto out_of_bounds; \
} while (0) } while (0)
#else
#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) goto out_of_bounds;
#endif
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
/* If memory64 is enabled, offset1, offset1 + bytes can \
* overflow */ \
if (disable_bounds_checks \
|| (offset1 >= offset && offset1 + bytes >= offset1 \
&& offset1 + bytes <= get_linear_mem_size())) \
maddr = memory->memory_data + offset1; \
else \
CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \ do { \
uint64 offset1 = (uint64)(start); \ uint64 offset1 = (uint64)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
/* If memory64 is enabled, offset1 + bytes can overflow */ \ /* If memory64 is enabled, offset1 + bytes can overflow */ \
if (disable_bounds_checks \ if (disable_bounds_checks \
|| (offset1 + bytes >= offset1 \ || (offset1 + bytes >= offset1 \
@ -153,7 +137,7 @@ typedef float64 CellType_F64;
bulk memory operation */ \ bulk memory operation */ \
maddr = memory->memory_data + offset1; \ maddr = memory->memory_data + offset1; \
else \ else \
CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes); \ goto out_of_bounds; \
} while (0) } while (0)
#endif /* end of WASM_ENABLE_MEMORY64 == 0 */ #endif /* end of WASM_ENABLE_MEMORY64 == 0 */
@ -1648,6 +1632,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (memory) if (memory)
is_memory64 = memory->is_memory64; is_memory64 = memory->is_memory64;
#endif #endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap = module->e->shared_heap;
uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL;
#if WASM_ENABLE_MEMORY64 != 0
uint64 shared_heap_start_off =
shared_heap ? (is_memory64 ? shared_heap->start_off_mem64
: shared_heap->start_off_mem32)
: 0;
uint64 shared_heap_end_off =
shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0;
#else
uint64 shared_heap_start_off =
shared_heap ? shared_heap->start_off_mem32 : 0;
uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0;
#endif
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
#if WASM_ENABLE_MULTI_MEMORY != 0 #if WASM_ENABLE_MULTI_MEMORY != 0
uint32 memidx = 0; uint32 memidx = 0;
uint32 memidx_cached = (uint32)-1; uint32 memidx_cached = (uint32)-1;
@ -3498,8 +3498,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
str_obj = (WASMString)wasm_stringref_obj_get_value( str_obj = (WASMString)wasm_stringref_obj_get_value(
stringref_obj); stringref_obj);
memory_inst = module->memories[mem_idx]; #if WASM_ENABLE_SHARED_HEAP != 0
maddr = memory_inst->memory_data + addr; if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
if (opcode == WASM_OP_STRING_ENCODE_WTF16) { if (opcode == WASM_OP_STRING_ENCODE_WTF16) {
flag = WTF16; flag = WTF16;
@ -3666,8 +3673,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
addr = POP_I32(); addr = POP_I32();
stringview_wtf8_obj = POP_REF(); stringview_wtf8_obj = POP_REF();
memory_inst = module->memories[mem_idx]; #if WASM_ENABLE_SHARED_HEAP != 0
maddr = memory_inst->memory_data + addr; if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
bytes_written = wasm_string_encode( bytes_written = wasm_string_encode(
(WASMString)wasm_stringview_wtf8_obj_get_value( (WASMString)wasm_stringview_wtf8_obj_get_value(
@ -5694,9 +5708,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr);
#else #else
if ((uint64)(uint32)addr + bytes > linear_mem_size) #if WASM_ENABLE_SHARED_HEAP != 0
goto out_of_bounds; if (app_addr_in_shared_heap((uint64)(uint32)addr,
maddr = memory->memory_data + (uint32)addr; bytes))
shared_heap_addr_app_to_native((uint64)(uint32)addr,
maddr);
else
#endif
{
if ((uint64)(uint32)addr + bytes > linear_mem_size)
goto out_of_bounds;
maddr = memory->memory_data + (uint32)addr;
}
#endif #endif
if (bh_bitmap_get_bit(module->e->common.data_dropped, if (bh_bitmap_get_bit(module->e->common.data_dropped,
@ -5746,15 +5769,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_THREAD_MGR != 0 #if WASM_ENABLE_THREAD_MGR != 0
linear_mem_size = get_linear_mem_size(); linear_mem_size = get_linear_mem_size();
#endif #endif
dlen = linear_mem_size - dst;
/* dst boundary check */ /* dst boundary check */
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else #if WASM_ENABLE_SHARED_HEAP != 0
if ((uint64)dst + len > linear_mem_size) if (app_addr_in_shared_heap((uint64)dst, len))
goto out_of_bounds; dlen = shared_heap_end_off - dst + 1;
mdst = memory->memory_data + dst;
#endif #endif
dlen = linear_mem_size - dst; #else /* else of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)dst, len)) {
shared_heap_addr_app_to_native((uint64)dst, mdst);
dlen = shared_heap_end_off - dst + 1;
}
else
#endif
{
if ((uint64)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + dst;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_MULTI_MEMORY != 0 #if WASM_ENABLE_MULTI_MEMORY != 0
/* src memidx */ /* src memidx */
@ -5770,9 +5808,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc);
#else #else
if ((uint64)src + len > linear_mem_size) #if WASM_ENABLE_SHARED_HEAP != 0
goto out_of_bounds; if (app_addr_in_shared_heap((uint64)src, len))
msrc = memory->memory_data + src; shared_heap_addr_app_to_native((uint64)src, msrc);
else
#endif
{
if ((uint64)src + len > linear_mem_size)
goto out_of_bounds;
msrc = memory->memory_data + src;
}
#endif #endif
#if WASM_ENABLE_MEMORY64 == 0 #if WASM_ENABLE_MEMORY64 == 0
@ -5809,9 +5854,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else #else
if ((uint64)(uint32)dst + len > linear_mem_size) #if WASM_ENABLE_SHARED_HEAP != 0
goto out_of_bounds; if (app_addr_in_shared_heap((uint64)(uint32)dst, len))
mdst = memory->memory_data + (uint32)dst; shared_heap_addr_app_to_native((uint64)(uint32)dst,
mdst);
else
#endif
{
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
}
#endif #endif
memset(mdst, fill_val, len); memset(mdst, fill_val, len);

168
core/iwasm/interpreter/wasm_interp_fast.c
View File

@ -37,29 +37,20 @@ typedef float64 CellType_F64;
#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory) #define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory)
#endif #endif
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 \
|| WASM_ENABLE_BULK_MEMORY != 0
#define GOTO_OUT_OF_BOUNDS goto out_of_bounds;
#else
#define GOTO_OUT_OF_BOUNDS
#endif
#if WASM_ENABLE_SHARED_HEAP != 0 #if WASM_ENABLE_SHARED_HEAP != 0
#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) \ #define app_addr_in_shared_heap(app_addr, bytes) \
do { \ (shared_heap && (app_addr) >= shared_heap_start_off \
if (offset1 + bytes >= UINT32_MAX - module->e->shared_heap->size \ && (app_addr) <= shared_heap_end_off - bytes + 1)
&& offset1 + bytes <= UINT32_MAX) { \
uint64 heap_start = UINT32_MAX - module->e->shared_heap->size; \ #define shared_heap_addr_app_to_native(app_addr, native_addr) \
uint64 heap_offset = (uint64)offset1 - heap_start; \ native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off)
maddr = module->e->shared_heap->base_addr + heap_offset; \
} \ #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \
else { \ if (app_addr_in_shared_heap(app_addr, bytes)) \
GOTO_OUT_OF_BOUNDS; \ shared_heap_addr_app_to_native(app_addr, native_addr); \
} \ else
} while (0)
#else #else
#define CHECK_MEMORY_SHARED_HEAP_OVERFLOW(bytes) GOTO_OUT_OF_BOUNDS #define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr)
#endif #endif
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \ #if !defined(OS_ENABLE_HW_BOUND_CHECK) \
@ -67,35 +58,39 @@ typedef float64 CellType_F64;
#define CHECK_MEMORY_OVERFLOW(bytes) \ #define CHECK_MEMORY_OVERFLOW(bytes) \
do { \ do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \ uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* If offset1 is in valid range, maddr must also \ /* If offset1 is in valid range, maddr must also \
be in valid range, no need to check it again. */ \ be in valid range, no need to check it again. */ \
maddr = memory->memory_data + offset1; \ maddr = memory->memory_data + offset1; \
else \ else \
CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ goto out_of_bounds; \
} while (0) } while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \ do { \
uint64 offset1 = (uint32)(start); \ uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \ if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* App heap space is not valid space for \ /* App heap space is not valid space for \
bulk memory operation */ \ bulk memory operation */ \
maddr = memory->memory_data + offset1; \ maddr = memory->memory_data + offset1; \
else \ else \
CHECK_MEMORY_SHARED_HEAP_OVERFLOW(byets); \ goto out_of_bounds; \
} while (0) } while (0)
#else #else
#define CHECK_MEMORY_OVERFLOW(bytes) \ #define CHECK_MEMORY_OVERFLOW(bytes) \
do { \ do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \ uint64 offset1 = (uint64)offset + (uint64)addr; \
maddr = memory->memory_data + offset1; \ CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0) } while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \ #define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \ do { \
maddr = memory->memory_data + (uint32)(start); \ uint64 offset1 = (uint32)(start); \
uint64 offset1 = start; \ CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0) } while (0)
#endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \ #endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */ || WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
@ -1542,6 +1537,24 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0 #if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
bool is_return_call = false; bool is_return_call = false;
#endif #endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL;
uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL;
/*
#if WASM_ENABLE_MEMORY64 != 0
uint64 shared_heap_start_off =
shared_heap ? (is_memory64 ? shared_heap->start_off_mem64
: shared_heap->start_off_mem32)
: 0;
uint64 shared_heap_end_off =
shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0;
#else
*/ /* TODO: uncomment the code when memory64 is enabled for fast-interp */
uint64 shared_heap_start_off =
shared_heap ? shared_heap->start_off_mem32 : 0;
uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0;
/* #endif */
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
#if WASM_ENABLE_LABELS_AS_VALUES != 0 #if WASM_ENABLE_LABELS_AS_VALUES != 0
#define HANDLE_OPCODE(op) &&HANDLE_##op #define HANDLE_OPCODE(op) &&HANDLE_##op
@ -2857,8 +2870,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
str_obj = (WASMString)wasm_stringref_obj_get_value( str_obj = (WASMString)wasm_stringref_obj_get_value(
stringref_obj); stringref_obj);
memory_inst = module->memories[mem_idx]; #if WASM_ENABLE_SHARED_HEAP != 0
maddr = memory_inst->memory_data + addr; if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
if (opcode == WASM_OP_STRING_ENCODE_WTF16) { if (opcode == WASM_OP_STRING_ENCODE_WTF16) {
flag = WTF16; flag = WTF16;
@ -3025,8 +3045,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
addr = POP_I32(); addr = POP_I32();
stringview_wtf8_obj = POP_REF(); stringview_wtf8_obj = POP_REF();
memory_inst = module->memories[mem_idx]; #if WASM_ENABLE_SHARED_HEAP != 0
maddr = memory_inst->memory_data + addr; if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
bytes_written = wasm_string_encode( bytes_written = wasm_string_encode(
(WASMString)wasm_stringview_wtf8_obj_get_value( (WASMString)wasm_stringview_wtf8_obj_get_value(
@ -5011,9 +5038,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr); CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr);
#else #else
if ((uint64)(uint32)addr + bytes > linear_mem_size) #if WASM_ENABLE_SHARED_HEAP != 0
goto out_of_bounds; if (app_addr_in_shared_heap((uint64)(uint32)addr,
maddr = memory->memory_data + (uint32)addr; bytes))
shared_heap_addr_app_to_native((uint64)(uint32)addr,
maddr);
else
#endif
{
if ((uint64)(uint32)addr + bytes > linear_mem_size)
goto out_of_bounds;
maddr = memory->memory_data + (uint32)addr;
}
#endif #endif
if (bh_bitmap_get_bit(module->e->common.data_dropped, if (bh_bitmap_get_bit(module->e->common.data_dropped,
segment)) { segment)) {
@ -5046,6 +5082,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
{ {
uint32 dst, src, len; uint32 dst, src, len;
uint8 *mdst, *msrc; uint8 *mdst, *msrc;
uint64 dlen;
len = POP_I32(); len = POP_I32();
src = POP_I32(); src = POP_I32();
@ -5055,22 +5092,43 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
linear_mem_size = get_linear_mem_size(); linear_mem_size = get_linear_mem_size();
#endif #endif
dlen = linear_mem_size - dst;
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc); CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc);
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else #if WASM_ENABLE_SHARED_HEAP != 0
if ((uint64)(uint32)src + len > linear_mem_size) if (app_addr_in_shared_heap((uint64)dst, len))
goto out_of_bounds; dlen = shared_heap_end_off - dst + 1;
msrc = memory->memory_data + (uint32)src;
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
#endif #endif
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)src, len))
shared_heap_addr_app_to_native((uint64)src, msrc);
else
#endif
{
if ((uint64)(uint32)src + len > linear_mem_size)
goto out_of_bounds;
msrc = memory->memory_data + (uint32)src;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)dst, len)) {
shared_heap_addr_app_to_native((uint64)dst, mdst);
dlen = shared_heap_end_off - dst + 1;
}
else
#endif
{
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
/* allowing the destination and source to overlap */ /* allowing the destination and source to overlap */
bh_memmove_s(mdst, (uint32)(linear_mem_size - dst), bh_memmove_s(mdst, (uint32)dlen, msrc, len);
msrc, len);
break; break;
} }
case WASM_OP_MEMORY_FILL: case WASM_OP_MEMORY_FILL:
@ -5089,9 +5147,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK #ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst); CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else #else
if ((uint64)(uint32)dst + len > linear_mem_size) #if WASM_ENABLE_SHARED_HEAP != 0
goto out_of_bounds; if (app_addr_in_shared_heap((uint64)(uint32)dst, len))
mdst = memory->memory_data + (uint32)dst; shared_heap_addr_app_to_native((uint64)(uint32)dst,
mdst);
else
#endif
{
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
}
#endif #endif
memset(mdst, fill_val, len); memset(mdst, fill_val, len);

4
core/iwasm/interpreter/wasm_runtime.h
View File

@ -96,8 +96,8 @@ typedef union {
typedef struct WASMSharedHeap { typedef struct WASMSharedHeap {
struct WASMSharedHeap *next; struct WASMSharedHeap *next;
void *heap_handle; void *heap_handle;
uint8_t *base_addr; uint8 *base_addr;
uint32_t size; uint32 size;
uint64 start_off_mem64; uint64 start_off_mem64;
uint64 start_off_mem32; uint64 start_off_mem32;
} WASMSharedHeap; } WASMSharedHeap;

9
core/iwasm/libraries/thread-mgr/thread_manager.c
View File

@ -1449,10 +1449,19 @@ wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
bh_assert(cluster); bh_assert(cluster);
os_mutex_lock(&cluster->lock); os_mutex_lock(&cluster->lock);
/* Try attaching shared heap to this module instance first
to ensure that we can attach it to all other instances. */
if (!wasm_runtime_attach_shared_heap_internal(module_inst, heap)) {
os_mutex_unlock(&cluster->lock);
return false;
}
/* Detach the shared heap so it can be attached again. */
wasm_runtime_detach_shared_heap_internal(module_inst);
traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor, traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor,
heap); heap);
os_mutex_unlock(&cluster->lock); os_mutex_unlock(&cluster->lock);
} }
return true; return true;
} }