christoph/wamr
1
Fork 0
Code Releases Activity

Merge pull request #3823 from bytecodealliance/dev/shared_heap

Browse Source 
Implement the shared heap feature for interpreter, aot and llvm jit.
Add below runtime APIs:
```C
wasm_shared_heap_t
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args);

bool
wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst,
                                wasm_shared_heap_t shared_heap);

void
wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst);

uint64_t
wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size,
                                void **p_native_addr);

void
wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr);
```

And allow wasm app to call API shared_heap_malloc and shared_heap_free:
```C
void *shared_heap_malloc(uint32_t size);
void shared_heap_free(void *ptr);
```
This commit is contained in:
Wenyong Huang
2024-10-15 14:26:22 +08:00
committed by GitHub
parent 0152e2c9f7 19160f0e1d
commit b038f2721b
41 changed files with 2499 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

575
core/iwasm/common/wasm_memory.c
View File

@ -13,6 +13,10 @@
#include "../common/wasm_shared_memory.h"
#endif
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif
typedef enum Memory_Mode {
MEMORY_MODE_UNKNOWN = 0,
MEMORY_MODE_POOL,
@ -24,6 +28,11 @@ static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN;
static mem_allocator_t pool_allocator = NULL;
#if WASM_ENABLE_SHARED_HEAP != 0
static WASMSharedHeap *shared_heap_list = NULL;
static korp_mutex shared_heap_list_lock;
#endif
static enlarge_memory_error_callback_t enlarge_memory_error_cb;
static void *enlarge_memory_error_user_data;
@ -132,16 +141,371 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst)
#endif
}
#if WASM_ENABLE_SHARED_HEAP != 0
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size);
static void
wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size,
uint64 map_size);
static void *
runtime_malloc(uint64 size)
{
void *mem;
if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) {
LOG_WARNING("Allocate memory failed");
return NULL;
}
memset(mem, 0, (uint32)size);
return mem;
}
WASMSharedHeap *
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)
{
uint64 heap_struct_size = sizeof(WASMSharedHeap), map_size;
uint32 size = init_args->size;
WASMSharedHeap *heap;
if (size == 0) {
goto fail1;
}
if (!(heap = runtime_malloc(heap_struct_size))) {
goto fail1;
}
if (!(heap->heap_handle =
runtime_malloc(mem_allocator_get_heap_struct_size()))) {
goto fail2;
}
size = align_uint(size, os_getpagesize());
heap->size = size;
heap->start_off_mem64 = UINT64_MAX - heap->size + 1;
heap->start_off_mem32 = UINT32_MAX - heap->size + 1;
if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) {
LOG_WARNING("Invalid size of shared heap");
goto fail3;
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = size;
#else
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
#endif
if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) {
goto fail3;
}
if (!mem_allocator_create_with_struct_and_pool(
heap->heap_handle, heap_struct_size, heap->base_addr, size)) {
LOG_WARNING("init share heap failed");
goto fail4;
}
os_mutex_lock(&shared_heap_list_lock);
if (shared_heap_list == NULL) {
shared_heap_list = heap;
}
else {
heap->next = shared_heap_list;
shared_heap_list = heap;
}
os_mutex_unlock(&shared_heap_list_lock);
return heap;
fail4:
wasm_munmap_linear_memory(heap->base_addr, size, map_size);
fail3:
wasm_runtime_free(heap->heap_handle);
fail2:
wasm_runtime_free(heap);
fail1:
return NULL;
}
bool
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
uint64 linear_mem_size;
if (!memory)
return false;
linear_mem_size = memory->memory_data_size;
/* check if linear memory and shared heap are overlapped */
if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64)
|| (!memory->is_memory64
&& linear_mem_size > shared_heap->start_off_mem32)) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
return false;
}
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
if (e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
e->shared_heap = shared_heap;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
if (memory->is_memory64)
e->shared_heap_start_off.u64 = shared_heap->start_off_mem64;
else
e->shared_heap_start_off.u64 = shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u64;
#else
e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u32[0];
#endif
#endif /* end of WASM_ENABLE_JIT != 0 */
}
#endif /* end of WASM_ENABLE_INTERP != 0 */
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
if (e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
e->shared_heap = shared_heap;
#if UINTPTR_MAX == UINT64_MAX
if (memory->is_memory64)
e->shared_heap_start_off.u64 = shared_heap->start_off_mem64;
else
e->shared_heap_start_off.u64 = shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u64;
#else
e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u32[0];
#endif
}
#endif /* end of WASM_ENABLE_AOT != 0 */
return true;
}
bool
wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap)
{
#if WASM_ENABLE_THREAD_MGR != 0
return wasm_cluster_attach_shared_heap(module_inst, shared_heap);
#else
return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap);
#endif
}
void
wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst)
{
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
e->shared_heap = NULL;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
#else
e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
e->shared_heap_base_addr_adj = NULL;
#endif
}
#endif /* end of WASM_ENABLE_INTERP != 0 */
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
e->shared_heap = NULL;
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
#else
e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
e->shared_heap_base_addr_adj = NULL;
}
#endif /* end of WASM_ENABLE_AOT != 0 */
}
void
wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst)
{
#if WASM_ENABLE_THREAD_MGR != 0
wasm_cluster_detach_shared_heap(module_inst);
#else
wasm_runtime_detach_shared_heap_internal(module_inst);
#endif
}
static WASMSharedHeap *
get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
{
#if WASM_ENABLE_INTERP != 0
if (module_inst_comm->module_type == Wasm_Module_Bytecode) {
return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst_comm->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_comm)
->e;
return e->shared_heap;
}
#endif
return NULL;
}
WASMSharedHeap *
wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
{
return get_shared_heap(module_inst_comm);
}
static bool
is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
bool is_memory64, uint64 app_offset, uint32 bytes)
{
WASMSharedHeap *heap = get_shared_heap(module_inst);
if (!heap) {
return false;
}
if (bytes == 0) {
bytes = 1;
}
if (!is_memory64) {
if (app_offset >= heap->start_off_mem32
&& app_offset <= UINT32_MAX - bytes + 1) {
return true;
}
}
else {
if (app_offset >= heap->start_off_mem64
&& app_offset <= UINT64_MAX - bytes + 1) {
return true;
}
}
return false;
}
static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes)
{
WASMSharedHeap *heap = get_shared_heap(module_inst);
if (heap && addr >= heap->base_addr
&& addr + bytes <= heap->base_addr + heap->size
&& addr + bytes > addr) {
return true;
}
return false;
}
uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64_t size, void **p_native_addr)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
WASMSharedHeap *shared_heap = get_shared_heap(module_inst);
void *native_addr = NULL;
if (!memory || !shared_heap)
return 0;
native_addr = mem_allocator_malloc(shared_heap->heap_handle, size);
if (!native_addr)
return 0;
if (p_native_addr) {
*p_native_addr = native_addr;
}
if (memory->is_memory64)
return shared_heap->start_off_mem64
+ ((uint8 *)native_addr - shared_heap->base_addr);
else
return shared_heap->start_off_mem32
+ ((uint8 *)native_addr - shared_heap->base_addr);
}
void
wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
WASMSharedHeap *shared_heap = get_shared_heap(module_inst);
uint8 *addr = NULL;
if (!memory || !shared_heap) {
return;
}
if (memory->is_memory64) {
if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */
LOG_WARNING("The address to free isn't in shared heap");
return;
}
addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64);
}
else {
if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) {
LOG_WARNING("The address to free isn't in shared heap");
return;
}
addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32);
}
mem_allocator_free(shared_heap->heap_handle, addr);
}
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
bool
wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
const MemAllocOption *alloc_option)
{
bool ret = false;
#if WASM_ENABLE_SHARED_HEAP != 0
if (os_mutex_init(&shared_heap_list_lock)) {
return false;
}
#endif
if (mem_alloc_type == Alloc_With_Pool) {
return wasm_memory_init_with_pool(alloc_option->pool.heap_buf,
alloc_option->pool.heap_size);
ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf,
alloc_option->pool.heap_size);
}
else if (mem_alloc_type == Alloc_With_Allocator) {
return wasm_memory_init_with_allocator(
ret = wasm_memory_init_with_allocator(
#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
alloc_option->allocator.user_data,
#endif
@ -151,16 +515,58 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
}
else if (mem_alloc_type == Alloc_With_System_Allocator) {
memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR;
return true;
ret = true;
}
else {
return false;
ret = false;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (!ret) {
os_mutex_destroy(&shared_heap_list_lock);
}
#endif
return ret;
}
#if WASM_ENABLE_SHARED_HEAP != 0
static void
destroy_shared_heaps()
{
WASMSharedHeap *heap;
WASMSharedHeap *cur;
uint64 map_size;
os_mutex_lock(&shared_heap_list_lock);
heap = shared_heap_list;
shared_heap_list = NULL;
os_mutex_unlock(&shared_heap_list_lock);
while (heap) {
cur = heap;
heap = heap->next;
mem_allocator_destroy(cur->heap_handle);
wasm_runtime_free(cur->heap_handle);
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = cur->size;
#else
map_size = 8 * (uint64)BH_GB;
#endif
wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size);
wasm_runtime_free(cur);
}
os_mutex_destroy(&shared_heap_list_lock);
}
#endif
void
wasm_runtime_memory_destroy(void)
{
#if WASM_ENABLE_SHARED_HEAP != 0
destroy_shared_heaps();
#endif
if (memory_mode == MEMORY_MODE_POOL) {
#if BH_ENABLE_GC_VERIFY == 0
(void)mem_allocator_destroy(pool_allocator);
@ -335,6 +741,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, size)) {
return true;
}
#endif
#if WASM_ENABLE_MEMORY64 != 0
if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
@ -364,6 +777,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
uint64 app_str_offset)
{
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
WASMMemoryInstance *memory_inst;
uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE;
char *str, *str_end;
@ -374,22 +788,42 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
return true;
}
if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL,
&app_end_offset))
memory_inst = wasm_get_default_memory(module_inst);
if (!memory_inst) {
goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_str_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
str = (char *)shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_str_offset - shared_heap->start_off_mem64)
: (app_str_offset - shared_heap->start_off_mem32));
str_end = (char *)shared_heap->base_addr + shared_heap->size;
}
else
#endif
{
if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset,
NULL, &app_end_offset))
goto fail;
#if WASM_ENABLE_MEMORY64 != 0
if (module_inst->memories[0]->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
#endif
/* boundary overflow check, max start offset can only be size - 1, while end
* offset can be size */
if (app_str_offset >= max_linear_memory_size
|| app_end_offset > max_linear_memory_size)
goto fail;
/* boundary overflow check, max start offset can be size - 1, while end
offset can be size */
if (app_str_offset >= max_linear_memory_size
|| app_end_offset > max_linear_memory_size)
goto fail;
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
str_end = str + (app_end_offset - app_str_offset);
}
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
str_end = str + (app_end_offset - app_str_offset);
while (str < str_end && *str != '\0')
str++;
if (str == str_end)
@ -431,6 +865,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) {
return true;
}
#endif
SHARED_MEMORY_LOCK(memory_inst);
if (memory_inst->memory_data <= addr
@ -465,6 +905,23 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
return NULL;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem32;
}
else if (memory_inst && memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem64;
}
return shared_heap->base_addr + app_offset - shared_heap_start;
}
#endif
SHARED_MEMORY_LOCK(memory_inst);
addr = memory_inst->memory_data + (uintptr_t)app_offset;
@ -499,11 +956,32 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
bounds_checks = is_bounds_checks_enabled(module_inst_comm);
#if WASM_ENABLE_SHARED_HEAP != 0
/* If shared heap is enabled, bounds check is always needed */
bounds_checks = true;
#endif
memory_inst = wasm_get_default_memory(module_inst);
if (!memory_inst) {
return 0;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem32;
}
else if (memory_inst && memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem64;
}
return shared_heap_start + (addr - shared_heap->base_addr);
}
#endif
SHARED_MEMORY_LOCK(memory_inst);
if (bounds_checks) {
@ -601,6 +1079,10 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst);
uint8 *native_addr;
bool bounds_checks;
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
bool is_in_shared_heap = false;
#endif
bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX);
@ -609,9 +1091,25 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
return false;
}
native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
memory_inst->is_memory64, app_buf_addr,
app_buf_size)) {
shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst);
native_addr = shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_buf_addr - shared_heap->start_off_mem64)
: (app_buf_addr - shared_heap->start_off_mem32));
is_in_shared_heap = true;
}
else
#endif
{
native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr;
}
bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst);
bounds_checks =
is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst);
if (!bounds_checks) {
if (app_buf_addr == 0) {
@ -620,6 +1118,24 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
goto success;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_in_shared_heap) {
const char *str, *str_end;
/* The whole string must be in the linear memory */
str = (const char *)native_addr;
str_end = (const char *)shared_heap->base_addr + shared_heap->size;
while (str < str_end && *str != '\0')
str++;
if (str == str_end) {
wasm_set_exception(module_inst, "out of bounds memory access");
return false;
}
else
goto success;
}
#endif
/* No need to check the app_offset and buf_size if memory access
boundary check with hardware trap is enabled */
#ifndef OS_ENABLE_HW_BOUND_CHECK
@ -749,7 +1265,7 @@ wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size,
}
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size)
wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size)
{
return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size);
}
@ -758,6 +1274,9 @@ static bool
wasm_enlarge_memory_internal(WASMModuleInstanceCommon *module,
WASMMemoryInstance *memory, uint32 inc_page_count)
{
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#endif
uint8 *memory_data_old, *memory_data_new, *heap_data_old;
uint32 num_bytes_per_page, heap_size;
uint32 cur_page_count, max_page_count, total_page_count;
@ -805,6 +1324,24 @@ wasm_enlarge_memory_internal(WASMModuleInstanceCommon *module,
goto return_func;
}
#if WASM_ENABLE_SHARED_HEAP != 0
shared_heap = get_shared_heap(module);
if (shared_heap) {
if (memory->is_memory64
&& total_size_new > shared_heap->start_off_mem64) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
ret = false;
goto return_func;
}
else if (!memory->is_memory64
&& total_size_new > shared_heap->start_off_mem32) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
ret = false;
goto return_func;
}
}
#endif
bh_assert(total_size_new
<= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64));

29
core/iwasm/common/wasm_memory.h
View File

@ -41,6 +41,35 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size)
#define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args);
bool
wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap);
bool
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap);
void
wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst);
void
wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst);
WASMSharedHeap *
wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm);
uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64 size, void **p_native_addr);
void
wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst,
uint64 ptr);
#endif
bool
wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
const MemAllocOption *alloc_option);

13
core/iwasm/common/wasm_native.c
View File

@ -33,6 +33,11 @@ uint32
get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis);
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
uint32
get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis);
#endif
uint32
get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis);
@ -512,6 +517,14 @@ wasm_native_init()
goto fail;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
n_native_symbols = get_lib_shared_heap_export_apis(&native_symbols);
if (n_native_symbols > 0
&& !wasm_native_register_natives("env", native_symbols,
n_native_symbols))
goto fail;
#endif
#if WASM_ENABLE_BASE_LIB != 0
n_native_symbols = get_base_lib_export_apis(&native_symbols);
if (n_native_symbols > 0

61
core/iwasm/common/wasm_runtime_common.c
View File

@ -185,6 +185,9 @@ static bool
is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst)
{
WASMMemoryInstance *memory_inst;
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#endif
uint8 *mapped_mem_start_addr = NULL;
uint8 *mapped_mem_end_addr = NULL;
uint32 i;
@ -202,6 +205,21 @@ is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst)
}
}
#if WASM_ENABLE_SHARED_HEAP != 0
shared_heap =
wasm_runtime_get_shared_heap((WASMModuleInstanceCommon *)module_inst);
if (shared_heap) {
mapped_mem_start_addr = shared_heap->base_addr;
mapped_mem_end_addr = shared_heap->base_addr + 8 * (uint64)BH_GB;
if (mapped_mem_start_addr <= (uint8 *)sig_addr
&& (uint8 *)sig_addr < mapped_mem_end_addr) {
/* The address which causes segmentation fault is inside
the shared heap's guard regions */
return true;
}
}
#endif
return false;
}
@ -4513,9 +4531,14 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
uint32 *argv, uint32 argc, uint32 *argv_ret)
{
WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env);
#if WASM_ENABLE_MEMORY64 != 0
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module);
bool is_memory64 = memory ? memory->is_memory64 : false;
#endif
typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *);
NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr;
uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64;
uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size;
uint32 *argv_src = argv, i, argc1, ptr_len;
uint32 arg_i32;
bool ret = false;
@ -4540,11 +4563,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
#endif
{
*(uint32 *)argv_dst = arg_i32 = *argv_src++;
/* TODO: memory64 if future there is a way for supporting
* wasm64 and wasm32 in libc at the same time, remove the
* macro control */
#if WASM_ENABLE_MEMORY64 == 0
if (signature) {
if (signature
#if WASM_ENABLE_MEMORY64 != 0
&& !is_memory64
#endif
) {
if (signature[i + 1] == '*') {
/* param is a pointer */
if (signature[i + 2] == '~')
@ -4573,17 +4596,18 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
module, (uint64)arg_i32);
}
}
#endif
break;
}
case VALUE_TYPE_I64:
#if WASM_ENABLE_MEMORY64 != 0
{
uint64 arg_i64;
PUT_I64_TO_ADDR((uint32 *)argv_dst,
GET_I64_FROM_ADDR(argv_src));
argv_src += 2;
arg_i64 = *argv_dst;
if (signature) {
if (signature && is_memory64) {
/* TODO: memory64 pointer with length need a new symbol
* to represent type i64, with '~' still represent i32
* length */
@ -4744,9 +4768,6 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
fail:
if (argv1 != argv_buf)
wasm_runtime_free(argv1);
#if WASM_ENABLE_MEMORY64 == 0
(void)arg_i64;
#endif
return ret;
}
@ -5670,6 +5691,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
uint32 *argv_ret)
{
WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env);
#if WASM_ENABLE_MEMORY64 != 0
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module);
bool is_memory64 = memory ? memory->is_memory64 : false;
#endif
uint64 argv_buf[32] = { 0 }, *argv1 = argv_buf, *ints, *stacks, size,
arg_i64;
uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0;
@ -5735,11 +5761,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i32 = *argv_src++;
arg_i64 = arg_i32;
/* TODO: memory64 if future there is a way for supporting
* wasm64 and wasm32 in libc at the same time, remove the
* macro control */
#if WASM_ENABLE_MEMORY64 == 0
if (signature) {
if (signature
#if WASM_ENABLE_MEMORY64 != 0
&& !is_memory64
#endif
) {
if (signature[i + 1] == '*') {
/* param is a pointer */
if (signature[i + 2] == '~')
@ -5766,7 +5792,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
module, (uint64)arg_i32);
}
}
#endif
if (n_ints < MAX_REG_INTS)
ints[n_ints++] = arg_i64;
else
@ -5778,7 +5803,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i64 = GET_I64_FROM_ADDR(argv_src);
argv_src += 2;
if (signature) {
if (signature && is_memory64) {
/* TODO: memory64 pointer with length need a new symbol
* to represent type i64, with '~' still represent i32
* length */

47
core/iwasm/common/wasm_shared_memory.c
View File

@ -8,6 +8,9 @@
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif
#if WASM_ENABLE_AOT != 0
#include "../aot/aot_runtime.h"
#endif
/*
* Note: this lock can be per memory.
@ -243,6 +246,31 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info,
destroy_wait_info(wait_info);
}
#if WASM_ENABLE_SHARED_HEAP != 0
static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes)
{
WASMSharedHeap *shared_heap = NULL;
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
shared_heap = ((WASMModuleInstance *)module_inst)->e->shared_heap;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
shared_heap = e->shared_heap;
}
#endif
return shared_heap && addr >= shared_heap->base_addr
&& addr + bytes <= shared_heap->base_addr + shared_heap->size;
}
#endif
uint32
wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address,
uint64 expect, int64 timeout, bool wait64)
@ -271,9 +299,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address,
}
shared_memory_lock(module_inst->memories[0]);
if ((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + (wait64 ? 8 : 4)
> module_inst->memories[0]->memory_data_end) {
if (
#if WASM_ENABLE_SHARED_HEAP != 0
/* not in shared heap */
!is_native_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
address, wait64 ? 8 : 4)
&&
#endif
/* and not in linear memory */
((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + (wait64 ? 8 : 4)
> module_inst->memories[0]->memory_data_end)) {
shared_memory_unlock(module_inst->memories[0]);
wasm_runtime_set_exception(module, "out of bounds memory access");
return -1;
@ -397,6 +433,11 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address,
shared_memory_lock(module_inst->memories[0]);
out_of_bounds =
#if WASM_ENABLE_SHARED_HEAP != 0
/* not in shared heap */
!is_native_addr_in_shared_heap(module, address, 4) &&
#endif
/* and not in linear memory */
((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end);
shared_memory_unlock(module_inst->memories[0]);