christoph/wamr
1
Fork 0
Code Releases Activity

Merge pull request #3823 from bytecodealliance/dev/shared_heap

Browse Source 
Implement the shared heap feature for interpreter, aot and llvm jit.
Add below runtime APIs:
```C
wasm_shared_heap_t
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args);

bool
wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst,
                                wasm_shared_heap_t shared_heap);

void
wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst);

uint64_t
wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size,
                                void **p_native_addr);

void
wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr);
```

And allow wasm app to call API shared_heap_malloc and shared_heap_free:
```C
void *shared_heap_malloc(uint32_t size);
void shared_heap_free(void *ptr);
```
This commit is contained in:
Wenyong Huang
2024-10-15 14:26:22 +08:00
committed by GitHub
parent 0152e2c9f7 19160f0e1d
commit b038f2721b
41 changed files with 2499 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
core/config.h
View File

@ -396,7 +396,9 @@
#define APP_HEAP_SIZE_DEFAULT (8 * 1024)
#endif
#define APP_HEAP_SIZE_MIN (256)
#define APP_HEAP_SIZE_MAX (512 * 1024 * 1024)
/* The ems memory allocator supports maximal heap size 1GB,
see ems_gc_internal.h */
#define APP_HEAP_SIZE_MAX (1024 * 1024 * 1024)
/* Default min/max gc heap size of each app */
#ifndef GC_HEAP_SIZE_DEFAULT
@ -692,4 +694,8 @@
#endif
#endif /* WASM_ENABLE_FUZZ_TEST != 0 */
#ifndef WASM_ENABLE_SHARED_HEAP
#define WASM_ENABLE_SHARED_HEAP 0
#endif
#endif /* end of _CONFIG_H_ */

21
core/iwasm/aot/aot_runtime.c
View File

@ -57,6 +57,9 @@ bh_static_assert(sizeof(AOTMemoryInstance) == 120);
bh_static_assert(offsetof(AOTTableInstance, elems) == 24);
bh_static_assert(offsetof(AOTModuleInstanceExtra, stack_sizes) == 0);
bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj)
== 8);
bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16);
bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3);
@ -1895,6 +1898,24 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent,
extra->stack_sizes =
aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL);
/*
* The AOT code checks whether the n bytes to access are in shared heap
* by checking whether the beginning address meets:
* addr >= start_off && addr <= end_off - n-bytes + 1
* where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g.,
* UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit
* target. To simplify the check, when shared heap is disabled, we set
* the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit
* target, so in the checking, the above formula will be false, we don't
* need to check whether the shared heap is enabled or not in the AOT
* code.
*/
#if UINTPTR_MAX == UINT64_MAX
extra->shared_heap_start_off.u64 = UINT64_MAX;
#else
extra->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
#if WASM_ENABLE_PERF_PROFILING != 0
total_size = sizeof(AOTFuncPerfProfInfo)
* ((uint64)module->import_func_count + module->func_count);

12
core/iwasm/aot/aot_runtime.h
View File

@ -111,6 +111,14 @@ typedef struct AOTFunctionInstance {
typedef struct AOTModuleInstanceExtra {
DefPointer(const uint32 *, stack_sizes);
/*
* Adjusted shared heap based addr to simple the calculation
* in the aot code. The value is:
* shared_heap->base_addr - shared_heap->start_off
*/
DefPointer(uint8 *, shared_heap_base_addr_adj);
MemBound shared_heap_start_off;
WASMModuleInstanceExtraCommon common;
AOTFunctionInstance **functions;
uint32 function_count;
@ -119,6 +127,10 @@ typedef struct AOTModuleInstanceExtra {
bh_list *sub_module_inst_list;
WASMModuleInstanceCommon **import_func_module_insts;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#endif
} AOTModuleInstanceExtra;
#if defined(BUILD_TARGET_X86_64) || defined(BUILD_TARGET_AMD_64)

575
core/iwasm/common/wasm_memory.c
View File

@ -13,6 +13,10 @@
#include "../common/wasm_shared_memory.h"
#endif
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif
typedef enum Memory_Mode {
MEMORY_MODE_UNKNOWN = 0,
MEMORY_MODE_POOL,
@ -24,6 +28,11 @@ static Memory_Mode memory_mode = MEMORY_MODE_UNKNOWN;
static mem_allocator_t pool_allocator = NULL;
#if WASM_ENABLE_SHARED_HEAP != 0
static WASMSharedHeap *shared_heap_list = NULL;
static korp_mutex shared_heap_list_lock;
#endif
static enlarge_memory_error_callback_t enlarge_memory_error_cb;
static void *enlarge_memory_error_user_data;
@ -132,16 +141,371 @@ is_bounds_checks_enabled(WASMModuleInstanceCommon *module_inst)
#endif
}
#if WASM_ENABLE_SHARED_HEAP != 0
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size);
static void
wasm_munmap_linear_memory(void *mapped_mem, uint64 commit_size,
uint64 map_size);
static void *
runtime_malloc(uint64 size)
{
void *mem;
if (size >= UINT32_MAX || !(mem = wasm_runtime_malloc((uint32)size))) {
LOG_WARNING("Allocate memory failed");
return NULL;
}
memset(mem, 0, (uint32)size);
return mem;
}
WASMSharedHeap *
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args)
{
uint64 heap_struct_size = sizeof(WASMSharedHeap), map_size;
uint32 size = init_args->size;
WASMSharedHeap *heap;
if (size == 0) {
goto fail1;
}
if (!(heap = runtime_malloc(heap_struct_size))) {
goto fail1;
}
if (!(heap->heap_handle =
runtime_malloc(mem_allocator_get_heap_struct_size()))) {
goto fail2;
}
size = align_uint(size, os_getpagesize());
heap->size = size;
heap->start_off_mem64 = UINT64_MAX - heap->size + 1;
heap->start_off_mem32 = UINT32_MAX - heap->size + 1;
if (size > APP_HEAP_SIZE_MAX || size < APP_HEAP_SIZE_MIN) {
LOG_WARNING("Invalid size of shared heap");
goto fail3;
}
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = size;
#else
/* Totally 8G is mapped, the opcode load/store address range is 0 to 8G:
* ea = i + memarg.offset
* both i and memarg.offset are u32 in range 0 to 4G
* so the range of ea is 0 to 8G
*/
map_size = 8 * (uint64)BH_GB;
#endif
if (!(heap->base_addr = wasm_mmap_linear_memory(map_size, size))) {
goto fail3;
}
if (!mem_allocator_create_with_struct_and_pool(
heap->heap_handle, heap_struct_size, heap->base_addr, size)) {
LOG_WARNING("init share heap failed");
goto fail4;
}
os_mutex_lock(&shared_heap_list_lock);
if (shared_heap_list == NULL) {
shared_heap_list = heap;
}
else {
heap->next = shared_heap_list;
shared_heap_list = heap;
}
os_mutex_unlock(&shared_heap_list_lock);
return heap;
fail4:
wasm_munmap_linear_memory(heap->base_addr, size, map_size);
fail3:
wasm_runtime_free(heap->heap_handle);
fail2:
wasm_runtime_free(heap);
fail1:
return NULL;
}
bool
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
uint64 linear_mem_size;
if (!memory)
return false;
linear_mem_size = memory->memory_data_size;
/* check if linear memory and shared heap are overlapped */
if ((memory->is_memory64 && linear_mem_size > shared_heap->start_off_mem64)
|| (!memory->is_memory64
&& linear_mem_size > shared_heap->start_off_mem32)) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
return false;
}
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
if (e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
e->shared_heap = shared_heap;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
if (memory->is_memory64)
e->shared_heap_start_off.u64 = shared_heap->start_off_mem64;
else
e->shared_heap_start_off.u64 = shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u64;
#else
e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u32[0];
#endif
#endif /* end of WASM_ENABLE_JIT != 0 */
}
#endif /* end of WASM_ENABLE_INTERP != 0 */
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
if (e->shared_heap) {
LOG_WARNING("A shared heap is already attached");
return false;
}
e->shared_heap = shared_heap;
#if UINTPTR_MAX == UINT64_MAX
if (memory->is_memory64)
e->shared_heap_start_off.u64 = shared_heap->start_off_mem64;
else
e->shared_heap_start_off.u64 = shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u64;
#else
e->shared_heap_start_off.u32[0] = (uint32)shared_heap->start_off_mem32;
e->shared_heap_base_addr_adj =
shared_heap->base_addr - e->shared_heap_start_off.u32[0];
#endif
}
#endif /* end of WASM_ENABLE_AOT != 0 */
return true;
}
bool
wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap)
{
#if WASM_ENABLE_THREAD_MGR != 0
return wasm_cluster_attach_shared_heap(module_inst, shared_heap);
#else
return wasm_runtime_attach_shared_heap_internal(module_inst, shared_heap);
#endif
}
void
wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst)
{
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
WASMModuleInstanceExtra *e =
(WASMModuleInstanceExtra *)((WASMModuleInstance *)module_inst)->e;
e->shared_heap = NULL;
#if WASM_ENABLE_JIT != 0
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
#else
e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
e->shared_heap_base_addr_adj = NULL;
#endif
}
#endif /* end of WASM_ENABLE_INTERP != 0 */
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
e->shared_heap = NULL;
#if UINTPTR_MAX == UINT64_MAX
e->shared_heap_start_off.u64 = UINT64_MAX;
#else
e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
e->shared_heap_base_addr_adj = NULL;
}
#endif /* end of WASM_ENABLE_AOT != 0 */
}
void
wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst)
{
#if WASM_ENABLE_THREAD_MGR != 0
wasm_cluster_detach_shared_heap(module_inst);
#else
wasm_runtime_detach_shared_heap_internal(module_inst);
#endif
}
static WASMSharedHeap *
get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
{
#if WASM_ENABLE_INTERP != 0
if (module_inst_comm->module_type == Wasm_Module_Bytecode) {
return ((WASMModuleInstance *)module_inst_comm)->e->shared_heap;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst_comm->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst_comm)
->e;
return e->shared_heap;
}
#endif
return NULL;
}
WASMSharedHeap *
wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm)
{
return get_shared_heap(module_inst_comm);
}
static bool
is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
bool is_memory64, uint64 app_offset, uint32 bytes)
{
WASMSharedHeap *heap = get_shared_heap(module_inst);
if (!heap) {
return false;
}
if (bytes == 0) {
bytes = 1;
}
if (!is_memory64) {
if (app_offset >= heap->start_off_mem32
&& app_offset <= UINT32_MAX - bytes + 1) {
return true;
}
}
else {
if (app_offset >= heap->start_off_mem64
&& app_offset <= UINT64_MAX - bytes + 1) {
return true;
}
}
return false;
}
static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes)
{
WASMSharedHeap *heap = get_shared_heap(module_inst);
if (heap && addr >= heap->base_addr
&& addr + bytes <= heap->base_addr + heap->size
&& addr + bytes > addr) {
return true;
}
return false;
}
uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64_t size, void **p_native_addr)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
WASMSharedHeap *shared_heap = get_shared_heap(module_inst);
void *native_addr = NULL;
if (!memory || !shared_heap)
return 0;
native_addr = mem_allocator_malloc(shared_heap->heap_handle, size);
if (!native_addr)
return 0;
if (p_native_addr) {
*p_native_addr = native_addr;
}
if (memory->is_memory64)
return shared_heap->start_off_mem64
+ ((uint8 *)native_addr - shared_heap->base_addr);
else
return shared_heap->start_off_mem32
+ ((uint8 *)native_addr - shared_heap->base_addr);
}
void
wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst, uint64 ptr)
{
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module_inst);
WASMSharedHeap *shared_heap = get_shared_heap(module_inst);
uint8 *addr = NULL;
if (!memory || !shared_heap) {
return;
}
if (memory->is_memory64) {
if (ptr < shared_heap->start_off_mem64) { /* ptr can not > UINT64_MAX */
LOG_WARNING("The address to free isn't in shared heap");
return;
}
addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem64);
}
else {
if (ptr < shared_heap->start_off_mem32 || ptr > UINT32_MAX) {
LOG_WARNING("The address to free isn't in shared heap");
return;
}
addr = shared_heap->base_addr + (ptr - shared_heap->start_off_mem32);
}
mem_allocator_free(shared_heap->heap_handle, addr);
}
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
bool
wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
const MemAllocOption *alloc_option)
{
bool ret = false;
#if WASM_ENABLE_SHARED_HEAP != 0
if (os_mutex_init(&shared_heap_list_lock)) {
return false;
}
#endif
if (mem_alloc_type == Alloc_With_Pool) {
return wasm_memory_init_with_pool(alloc_option->pool.heap_buf,
alloc_option->pool.heap_size);
ret = wasm_memory_init_with_pool(alloc_option->pool.heap_buf,
alloc_option->pool.heap_size);
}
else if (mem_alloc_type == Alloc_With_Allocator) {
return wasm_memory_init_with_allocator(
ret = wasm_memory_init_with_allocator(
#if WASM_MEM_ALLOC_WITH_USER_DATA != 0
alloc_option->allocator.user_data,
#endif
@ -151,16 +515,58 @@ wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
}
else if (mem_alloc_type == Alloc_With_System_Allocator) {
memory_mode = MEMORY_MODE_SYSTEM_ALLOCATOR;
return true;
ret = true;
}
else {
return false;
ret = false;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (!ret) {
os_mutex_destroy(&shared_heap_list_lock);
}
#endif
return ret;
}
#if WASM_ENABLE_SHARED_HEAP != 0
static void
destroy_shared_heaps()
{
WASMSharedHeap *heap;
WASMSharedHeap *cur;
uint64 map_size;
os_mutex_lock(&shared_heap_list_lock);
heap = shared_heap_list;
shared_heap_list = NULL;
os_mutex_unlock(&shared_heap_list_lock);
while (heap) {
cur = heap;
heap = heap->next;
mem_allocator_destroy(cur->heap_handle);
wasm_runtime_free(cur->heap_handle);
#ifndef OS_ENABLE_HW_BOUND_CHECK
map_size = cur->size;
#else
map_size = 8 * (uint64)BH_GB;
#endif
wasm_munmap_linear_memory(cur->base_addr, cur->size, map_size);
wasm_runtime_free(cur);
}
os_mutex_destroy(&shared_heap_list_lock);
}
#endif
void
wasm_runtime_memory_destroy(void)
{
#if WASM_ENABLE_SHARED_HEAP != 0
destroy_shared_heaps();
#endif
if (memory_mode == MEMORY_MODE_POOL) {
#if BH_ENABLE_GC_VERIFY == 0
(void)mem_allocator_destroy(pool_allocator);
@ -335,6 +741,13 @@ wasm_runtime_validate_app_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, size)) {
return true;
}
#endif
#if WASM_ENABLE_MEMORY64 != 0
if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
@ -364,6 +777,7 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
uint64 app_str_offset)
{
WASMModuleInstance *module_inst = (WASMModuleInstance *)module_inst_comm;
WASMMemoryInstance *memory_inst;
uint64 app_end_offset, max_linear_memory_size = MAX_LINEAR_MEMORY_SIZE;
char *str, *str_end;
@ -374,22 +788,42 @@ wasm_runtime_validate_app_str_addr(WASMModuleInstanceCommon *module_inst_comm,
return true;
}
if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset, NULL,
&app_end_offset))
memory_inst = wasm_get_default_memory(module_inst);
if (!memory_inst) {
goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_str_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
str = (char *)shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_str_offset - shared_heap->start_off_mem64)
: (app_str_offset - shared_heap->start_off_mem32));
str_end = (char *)shared_heap->base_addr + shared_heap->size;
}
else
#endif
{
if (!wasm_runtime_get_app_addr_range(module_inst_comm, app_str_offset,
NULL, &app_end_offset))
goto fail;
#if WASM_ENABLE_MEMORY64 != 0
if (module_inst->memories[0]->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
if (memory_inst->is_memory64)
max_linear_memory_size = MAX_LINEAR_MEM64_MEMORY_SIZE;
#endif
/* boundary overflow check, max start offset can only be size - 1, while end
* offset can be size */
if (app_str_offset >= max_linear_memory_size
|| app_end_offset > max_linear_memory_size)
goto fail;
/* boundary overflow check, max start offset can be size - 1, while end
offset can be size */
if (app_str_offset >= max_linear_memory_size
|| app_end_offset > max_linear_memory_size)
goto fail;
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
str_end = str + (app_end_offset - app_str_offset);
}
str = wasm_runtime_addr_app_to_native(module_inst_comm, app_str_offset);
str_end = str + (app_end_offset - app_str_offset);
while (str < str_end && *str != '\0')
str++;
if (str == str_end)
@ -431,6 +865,12 @@ wasm_runtime_validate_native_addr(WASMModuleInstanceCommon *module_inst_comm,
goto fail;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, native_ptr, size)) {
return true;
}
#endif
SHARED_MEMORY_LOCK(memory_inst);
if (memory_inst->memory_data <= addr
@ -465,6 +905,23 @@ wasm_runtime_addr_app_to_native(WASMModuleInstanceCommon *module_inst_comm,
return NULL;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64,
app_offset, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem32;
}
else if (memory_inst && memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem64;
}
return shared_heap->base_addr + app_offset - shared_heap_start;
}
#endif
SHARED_MEMORY_LOCK(memory_inst);
addr = memory_inst->memory_data + (uintptr_t)app_offset;
@ -499,11 +956,32 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm,
bounds_checks = is_bounds_checks_enabled(module_inst_comm);
#if WASM_ENABLE_SHARED_HEAP != 0
/* If shared heap is enabled, bounds check is always needed */
bounds_checks = true;
#endif
memory_inst = wasm_get_default_memory(module_inst);
if (!memory_inst) {
return 0;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_native_addr_in_shared_heap(module_inst_comm, addr, 1)) {
WASMSharedHeap *shared_heap = get_shared_heap(module_inst_comm);
uint64 shared_heap_start = 0;
if (memory_inst && !memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem32;
}
else if (memory_inst && memory_inst->is_memory64) {
shared_heap_start = shared_heap->start_off_mem64;
}
return shared_heap_start + (addr - shared_heap->base_addr);
}
#endif
SHARED_MEMORY_LOCK(memory_inst);
if (bounds_checks) {
@ -601,6 +1079,10 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
WASMMemoryInstance *memory_inst = wasm_get_default_memory(module_inst);
uint8 *native_addr;
bool bounds_checks;
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
bool is_in_shared_heap = false;
#endif
bh_assert(app_buf_addr <= UINTPTR_MAX && app_buf_size <= UINTPTR_MAX);
@ -609,9 +1091,25 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
return false;
}
native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_app_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
memory_inst->is_memory64, app_buf_addr,
app_buf_size)) {
shared_heap = get_shared_heap((WASMModuleInstanceCommon *)module_inst);
native_addr = shared_heap->base_addr
+ (memory_inst->is_memory64
? (app_buf_addr - shared_heap->start_off_mem64)
: (app_buf_addr - shared_heap->start_off_mem32));
is_in_shared_heap = true;
}
else
#endif
{
native_addr = memory_inst->memory_data + (uintptr_t)app_buf_addr;
}
bounds_checks = is_bounds_checks_enabled((wasm_module_inst_t)module_inst);
bounds_checks =
is_bounds_checks_enabled((WASMModuleInstanceCommon *)module_inst);
if (!bounds_checks) {
if (app_buf_addr == 0) {
@ -620,6 +1118,24 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str,
goto success;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (is_in_shared_heap) {
const char *str, *str_end;
/* The whole string must be in the linear memory */
str = (const char *)native_addr;
str_end = (const char *)shared_heap->base_addr + shared_heap->size;
while (str < str_end && *str != '\0')
str++;
if (str == str_end) {
wasm_set_exception(module_inst, "out of bounds memory access");
return false;
}
else
goto success;
}
#endif
/* No need to check the app_offset and buf_size if memory access
boundary check with hardware trap is enabled */
#ifndef OS_ENABLE_HW_BOUND_CHECK
@ -749,7 +1265,7 @@ wasm_mremap_linear_memory(void *mapped_mem, uint64 old_size, uint64 new_size,
}
static void *
wasm_mmap_linear_memory(uint64_t map_size, uint64 commit_size)
wasm_mmap_linear_memory(uint64 map_size, uint64 commit_size)
{
return wasm_mremap_linear_memory(NULL, 0, map_size, commit_size);
}
@ -758,6 +1274,9 @@ static bool
wasm_enlarge_memory_internal(WASMModuleInstanceCommon *module,
WASMMemoryInstance *memory, uint32 inc_page_count)
{
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#endif
uint8 *memory_data_old, *memory_data_new, *heap_data_old;
uint32 num_bytes_per_page, heap_size;
uint32 cur_page_count, max_page_count, total_page_count;
@ -805,6 +1324,24 @@ wasm_enlarge_memory_internal(WASMModuleInstanceCommon *module,
goto return_func;
}
#if WASM_ENABLE_SHARED_HEAP != 0
shared_heap = get_shared_heap(module);
if (shared_heap) {
if (memory->is_memory64
&& total_size_new > shared_heap->start_off_mem64) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
ret = false;
goto return_func;
}
else if (!memory->is_memory64
&& total_size_new > shared_heap->start_off_mem32) {
LOG_WARNING("Linear memory address is overlapped with shared heap");
ret = false;
goto return_func;
}
}
#endif
bh_assert(total_size_new
<= GET_MAX_LINEAR_MEMORY_SIZE(memory->is_memory64));

29
core/iwasm/common/wasm_memory.h
View File

@ -41,6 +41,35 @@ SET_LINEAR_MEMORY_SIZE(WASMMemoryInstance *memory, uint64 size)
#define SET_LINEAR_MEMORY_SIZE(memory, size) memory->memory_data_size = size
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args);
bool
wasm_runtime_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap);
bool
wasm_runtime_attach_shared_heap_internal(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *shared_heap);
void
wasm_runtime_detach_shared_heap(WASMModuleInstanceCommon *module_inst);
void
wasm_runtime_detach_shared_heap_internal(WASMModuleInstanceCommon *module_inst);
WASMSharedHeap *
wasm_runtime_get_shared_heap(WASMModuleInstanceCommon *module_inst_comm);
uint64
wasm_runtime_shared_heap_malloc(WASMModuleInstanceCommon *module_inst,
uint64 size, void **p_native_addr);
void
wasm_runtime_shared_heap_free(WASMModuleInstanceCommon *module_inst,
uint64 ptr);
#endif
bool
wasm_runtime_memory_init(mem_alloc_type_t mem_alloc_type,
const MemAllocOption *alloc_option);

13
core/iwasm/common/wasm_native.c
View File

@ -33,6 +33,11 @@ uint32
get_spectest_export_apis(NativeSymbol **p_libc_builtin_apis);
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
uint32
get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis);
#endif
uint32
get_libc_wasi_export_apis(NativeSymbol **p_libc_wasi_apis);
@ -512,6 +517,14 @@ wasm_native_init()
goto fail;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
n_native_symbols = get_lib_shared_heap_export_apis(&native_symbols);
if (n_native_symbols > 0
&& !wasm_native_register_natives("env", native_symbols,
n_native_symbols))
goto fail;
#endif
#if WASM_ENABLE_BASE_LIB != 0
n_native_symbols = get_base_lib_export_apis(&native_symbols);
if (n_native_symbols > 0

61
core/iwasm/common/wasm_runtime_common.c
View File

@ -185,6 +185,9 @@ static bool
is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst)
{
WASMMemoryInstance *memory_inst;
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#endif
uint8 *mapped_mem_start_addr = NULL;
uint8 *mapped_mem_end_addr = NULL;
uint32 i;
@ -202,6 +205,21 @@ is_sig_addr_in_guard_pages(void *sig_addr, WASMModuleInstance *module_inst)
}
}
#if WASM_ENABLE_SHARED_HEAP != 0
shared_heap =
wasm_runtime_get_shared_heap((WASMModuleInstanceCommon *)module_inst);
if (shared_heap) {
mapped_mem_start_addr = shared_heap->base_addr;
mapped_mem_end_addr = shared_heap->base_addr + 8 * (uint64)BH_GB;
if (mapped_mem_start_addr <= (uint8 *)sig_addr
&& (uint8 *)sig_addr < mapped_mem_end_addr) {
/* The address which causes segmentation fault is inside
the shared heap's guard regions */
return true;
}
}
#endif
return false;
}
@ -4513,9 +4531,14 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
uint32 *argv, uint32 argc, uint32 *argv_ret)
{
WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env);
#if WASM_ENABLE_MEMORY64 != 0
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module);
bool is_memory64 = memory ? memory->is_memory64 : false;
#endif
typedef void (*NativeRawFuncPtr)(WASMExecEnv *, uint64 *);
NativeRawFuncPtr invoke_native_raw = (NativeRawFuncPtr)func_ptr;
uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size, arg_i64;
uint64 argv_buf[16] = { 0 }, *argv1 = argv_buf, *argv_dst, size;
uint32 *argv_src = argv, i, argc1, ptr_len;
uint32 arg_i32;
bool ret = false;
@ -4540,11 +4563,11 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
#endif
{
*(uint32 *)argv_dst = arg_i32 = *argv_src++;
/* TODO: memory64 if future there is a way for supporting
* wasm64 and wasm32 in libc at the same time, remove the
* macro control */
#if WASM_ENABLE_MEMORY64 == 0
if (signature) {
if (signature
#if WASM_ENABLE_MEMORY64 != 0
&& !is_memory64
#endif
) {
if (signature[i + 1] == '*') {
/* param is a pointer */
if (signature[i + 2] == '~')
@ -4573,17 +4596,18 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
module, (uint64)arg_i32);
}
}
#endif
break;
}
case VALUE_TYPE_I64:
#if WASM_ENABLE_MEMORY64 != 0
{
uint64 arg_i64;
PUT_I64_TO_ADDR((uint32 *)argv_dst,
GET_I64_FROM_ADDR(argv_src));
argv_src += 2;
arg_i64 = *argv_dst;
if (signature) {
if (signature && is_memory64) {
/* TODO: memory64 pointer with length need a new symbol
* to represent type i64, with '~' still represent i32
* length */
@ -4744,9 +4768,6 @@ wasm_runtime_invoke_native_raw(WASMExecEnv *exec_env, void *func_ptr,
fail:
if (argv1 != argv_buf)
wasm_runtime_free(argv1);
#if WASM_ENABLE_MEMORY64 == 0
(void)arg_i64;
#endif
return ret;
}
@ -5670,6 +5691,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
uint32 *argv_ret)
{
WASMModuleInstanceCommon *module = wasm_runtime_get_module_inst(exec_env);
#if WASM_ENABLE_MEMORY64 != 0
WASMMemoryInstance *memory =
wasm_get_default_memory((WASMModuleInstance *)module);
bool is_memory64 = memory ? memory->is_memory64 : false;
#endif
uint64 argv_buf[32] = { 0 }, *argv1 = argv_buf, *ints, *stacks, size,
arg_i64;
uint32 *argv_src = argv, i, argc1, n_ints = 0, n_stacks = 0;
@ -5735,11 +5761,11 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i32 = *argv_src++;
arg_i64 = arg_i32;
/* TODO: memory64 if future there is a way for supporting
* wasm64 and wasm32 in libc at the same time, remove the
* macro control */
#if WASM_ENABLE_MEMORY64 == 0
if (signature) {
if (signature
#if WASM_ENABLE_MEMORY64 != 0
&& !is_memory64
#endif
) {
if (signature[i + 1] == '*') {
/* param is a pointer */
if (signature[i + 2] == '~')
@ -5766,7 +5792,6 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
module, (uint64)arg_i32);
}
}
#endif
if (n_ints < MAX_REG_INTS)
ints[n_ints++] = arg_i64;
else
@ -5778,7 +5803,7 @@ wasm_runtime_invoke_native(WASMExecEnv *exec_env, void *func_ptr,
{
arg_i64 = GET_I64_FROM_ADDR(argv_src);
argv_src += 2;
if (signature) {
if (signature && is_memory64) {
/* TODO: memory64 pointer with length need a new symbol
* to represent type i64, with '~' still represent i32
* length */

47
core/iwasm/common/wasm_shared_memory.c
View File

@ -8,6 +8,9 @@
#if WASM_ENABLE_THREAD_MGR != 0
#include "../libraries/thread-mgr/thread_manager.h"
#endif
#if WASM_ENABLE_AOT != 0
#include "../aot/aot_runtime.h"
#endif
/*
* Note: this lock can be per memory.
@ -243,6 +246,31 @@ map_try_release_wait_info(HashMap *wait_hash_map, AtomicWaitInfo *wait_info,
destroy_wait_info(wait_info);
}
#if WASM_ENABLE_SHARED_HEAP != 0
static bool
is_native_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst,
uint8 *addr, uint32 bytes)
{
WASMSharedHeap *shared_heap = NULL;
#if WASM_ENABLE_INTERP != 0
if (module_inst->module_type == Wasm_Module_Bytecode) {
shared_heap = ((WASMModuleInstance *)module_inst)->e->shared_heap;
}
#endif
#if WASM_ENABLE_AOT != 0
if (module_inst->module_type == Wasm_Module_AoT) {
AOTModuleInstanceExtra *e =
(AOTModuleInstanceExtra *)((AOTModuleInstance *)module_inst)->e;
shared_heap = e->shared_heap;
}
#endif
return shared_heap && addr >= shared_heap->base_addr
&& addr + bytes <= shared_heap->base_addr + shared_heap->size;
}
#endif
uint32
wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address,
uint64 expect, int64 timeout, bool wait64)
@ -271,9 +299,17 @@ wasm_runtime_atomic_wait(WASMModuleInstanceCommon *module, void *address,
}
shared_memory_lock(module_inst->memories[0]);
if ((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + (wait64 ? 8 : 4)
> module_inst->memories[0]->memory_data_end) {
if (
#if WASM_ENABLE_SHARED_HEAP != 0
/* not in shared heap */
!is_native_addr_in_shared_heap((WASMModuleInstanceCommon *)module_inst,
address, wait64 ? 8 : 4)
&&
#endif
/* and not in linear memory */
((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + (wait64 ? 8 : 4)
> module_inst->memories[0]->memory_data_end)) {
shared_memory_unlock(module_inst->memories[0]);
wasm_runtime_set_exception(module, "out of bounds memory access");
return -1;
@ -397,6 +433,11 @@ wasm_runtime_atomic_notify(WASMModuleInstanceCommon *module, void *address,
shared_memory_lock(module_inst->memories[0]);
out_of_bounds =
#if WASM_ENABLE_SHARED_HEAP != 0
/* not in shared heap */
!is_native_addr_in_shared_heap(module, address, 4) &&
#endif
/* and not in linear memory */
((uint8 *)address < module_inst->memories[0]->memory_data
|| (uint8 *)address + 4 > module_inst->memories[0]->memory_data_end);
shared_memory_unlock(module_inst->memories[0]);

278
core/iwasm/compilation/aot_emit_memory.c
View File

@ -118,10 +118,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
{
LLVMValueRef offset_const =
MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset));
LLVMValueRef addr, maddr, offset1, cmp1, cmp2, cmp;
LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp;
LLVMValueRef mem_base_addr, mem_check_bound;
LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMBasicBlockRef check_succ;
LLVMBasicBlockRef check_succ, block_maddr_phi = NULL;
AOTValue *aot_value_top;
uint32 local_idx_of_aot_value = 0;
uint64 const_value;
@ -131,6 +131,11 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
bool is_shared_memory =
comp_ctx->comp_data->memories[0].flags & SHARED_MEMORY_FLAG;
#endif
#if WASM_ENABLE_MEMORY64 == 0
bool is_memory64 = false;
#else
bool is_memory64 = IS_MEMORY64;
#endif
is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false;
@ -268,8 +273,137 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
}
/* offset1 = offset + addr; */
/* TODO: check whether integer overflow occurs when memory is 64-bit
and boundary check is enabled */
BUILD_OP(Add, offset_const, addr, offset1, "offset1");
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem;
LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL;
/* Add basic blocks */
ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap");
ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem");
ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi");
LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr);
LLVMMoveBasicBlockAfter(app_addr_in_linear_mem,
app_addr_in_shared_heap);
LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
if (!(maddr_phi =
LLVMBuildPhi(comp_ctx->builder,
enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE,
"maddr_phi"))) {
aot_set_last_error("llvm build phi failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
if (!is_target_64bit) {
/* Check whether interger overflow occurs in addr + offset */
LLVMBasicBlockRef check_integer_overflow_end;
ADD_BASIC_BLOCK(check_integer_overflow_end,
"check_integer_overflow_end");
LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr);
BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1");
if (!aot_emit_exception(comp_ctx, func_ctx,
EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true,
cmp1, check_integer_overflow_end)) {
goto fail;
}
SET_BUILD_POS(check_integer_overflow_end);
}
shared_heap_check_bound =
is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1)
: (comp_ctx->pointer_size == sizeof(uint64)
? I64_CONST(UINT32_MAX - bytes + 1)
: I32_CONST(UINT32_MAX - bytes + 1));
CHECK_LLVM_CONST(shared_heap_check_bound);
/* Check whether the bytes to access are in shared heap */
if (!comp_ctx->enable_bound_check) {
/* Use IntUGT but not IntUGE to compare, since (1) in the ems
memory allocator, the hmu node includes hmu header and hmu
memory, only the latter is returned to the caller as the
allocated memory, the hmu header isn't returned so the
first byte of the shared heap won't be accesed, (2) using
IntUGT gets better performance than IntUGE in some cases */
BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off,
is_in_shared_heap, "is_in_shared_heap");
/* We don't check the shared heap's upper boundary if boundary
check isn't enabled, the runtime may also use the guard pages
of shared heap to check the boundary if hardware boundary
check feature is enabled. */
}
else {
/* Use IntUGT but not IntUGE to compare, same as above */
BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off,
cmp1, "cmp1");
/* Check the shared heap's upper boundary if boundary check is
enabled */
BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2,
"cmp2");
BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap");
}
if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap,
app_addr_in_shared_heap, app_addr_in_linear_mem)) {
aot_set_last_error("llvm build cond br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap);
/* Get native address inside shared heap */
if (!(maddr =
LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->shared_heap_base_addr_adj,
&offset1, 1, "maddr_shared_heap"))) {
aot_set_last_error("llvm build inbounds gep failed");
goto fail;
}
if (enable_segue) {
LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base;
if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr,
I64_TYPE, "maddr_u64"))
|| !(mem_base_addr_u64 =
LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr,
I64_TYPE, "mem_base_addr_u64"))) {
aot_set_last_error("llvm build ptr to int failed");
goto fail;
}
if (!(offset_to_mem_base =
LLVMBuildSub(comp_ctx->builder, maddr_u64,
mem_base_addr_u64, "offset_to_mem_base"))) {
aot_set_last_error("llvm build sub failed");
goto fail;
}
if (!(maddr = LLVMBuildIntToPtr(
comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS,
"maddr_shared_heap_segue"))) {
aot_set_last_error("llvm build int to ptr failed.");
goto fail;
}
}
LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem);
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
}
if (comp_ctx->enable_bound_check
&& !(is_local_of_aot_value
&& aot_checked_addr_list_find(func_ctx, local_idx_of_aot_value,
@ -305,10 +439,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp");
}
else {
/* Check integer overflow */
BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1");
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2");
BUILD_OP(Or, cmp1, cmp2, cmp, "cmp");
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
/* Check integer overflow has been checked above */
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp");
}
else {
/* Check integer overflow */
BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1");
BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2");
BUILD_OP(Or, cmp1, cmp2, cmp, "cmp");
}
}
/* Add basic blocks */
@ -354,7 +494,19 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
goto fail;
}
}
return maddr;
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
return maddr_phi;
}
else
return maddr;
fail:
return NULL;
}
@ -985,10 +1137,15 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
LLVMValueRef offset, LLVMValueRef bytes)
{
LLVMValueRef maddr, max_addr, cmp;
LLVMValueRef mem_base_addr;
LLVMValueRef mem_base_addr, maddr_phi = NULL;
LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMBasicBlockRef check_succ;
LLVMBasicBlockRef check_succ, block_maddr_phi = NULL;
LLVMValueRef mem_size;
#if WASM_ENABLE_MEMORY64 == 0
bool is_memory64 = false;
#else
bool is_memory64 = IS_MEMORY64;
#endif
/* Get memory base address and memory data size */
#if WASM_ENABLE_SHARED_MEMORY != 0
@ -1053,9 +1210,96 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
offset =
LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset");
bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len");
if (!offset || !bytes) {
aot_set_last_error("llvm build zext failed.");
goto fail;
}
/* TODO: check whether integer overflow occurs when memory is 64-bit
and boundary check is enabled */
BUILD_OP(Add, offset, bytes, max_addr, "max_addr");
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem;
LLVMValueRef shared_heap_start_off, shared_heap_check_bound;
LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap;
/* Add basic blocks */
ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap");
ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem");
ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi");
LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr);
LLVMMoveBasicBlockAfter(app_addr_in_linear_mem,
app_addr_in_shared_heap);
LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ);
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE,
"maddr_phi"))) {
aot_set_last_error("llvm build phi failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr);
shared_heap_start_off = func_ctx->shared_heap_start_off;
if (comp_ctx->pointer_size == sizeof(uint32)) {
if (!(shared_heap_start_off =
LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off,
I64_TYPE, "shared_heap_start_off_u64"))) {
aot_set_last_error("llvm build zext failed");
goto fail;
}
}
shared_heap_check_bound =
is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX);
CHECK_LLVM_CONST(shared_heap_check_bound);
/* Check whether the bytes to access are in shared heap */
if (!comp_ctx->enable_bound_check) {
/* Use IntUGT but not IntUGE to compare, same as the check
in aot_check_memory_overflow */
BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off,
is_in_shared_heap, "is_in_shared_heap");
}
else {
BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off,
cmp1, "cmp1");
BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset");
BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2,
"cmp2");
BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap");
}
if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap,
app_addr_in_shared_heap, app_addr_in_linear_mem)) {
aot_set_last_error("llvm build cond br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap);
/* Get native address inside shared heap */
if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->shared_heap_base_addr_adj,
&offset, 1, "maddr_shared_heap"))) {
aot_set_last_error("llvm build inbounds gep failed");
goto fail;
}
LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem);
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
}
BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr");
if (!aot_emit_exception(comp_ctx, func_ctx,
EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp,
check_succ)) {
@ -1068,11 +1312,23 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
aot_set_last_error("llvm build add failed.");
goto fail;
}
return maddr;
if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) {
block_curr = LLVMGetInsertBlock(comp_ctx->builder);
LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1);
if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) {
aot_set_last_error("llvm build br failed");
goto fail;
}
LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi);
return maddr_phi;
}
else
return maddr;
fail:
return NULL;
}
#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 or WASM_ENABLE_STRINGREF != 0 */
#endif /* end of WASM_ENABLE_BULK_MEMORY != 0 || WASM_ENABLE_STRINGREF != 0 */
#if WASM_ENABLE_BULK_MEMORY != 0
bool

78
core/iwasm/compilation/aot_llvm.c
View File

@ -1518,6 +1518,75 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
return true;
}
static bool
create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
LLVMValueRef offset, base_addr_p, start_off_p, cmp;
uint32 offset_u32;
/* Load aot_inst->e->shared_heap_base_addr_adj */
offset_u32 = get_module_inst_extra_offset(comp_ctx);
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0
if (comp_ctx->is_jit_mode)
offset_u32 +=
offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj);
else
#endif
offset_u32 +=
offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj);
offset = I32_CONST(offset_u32);
CHECK_LLVM_CONST(offset);
if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
"shared_heap_base_addr_adj_p"))) {
aot_set_last_error("llvm build inbounds gep failed");
return false;
}
if (!(func_ctx->shared_heap_base_addr_adj =
LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p,
"shared_heap_base_addr_adj"))) {
aot_set_last_error("llvm build load failed");
return false;
}
/* Load aot_inst->e->shared_heap_start_off */
offset_u32 = get_module_inst_extra_offset(comp_ctx);
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0
if (comp_ctx->is_jit_mode)
offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off);
else
#endif
offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off);
offset = I32_CONST(offset_u32);
CHECK_LLVM_CONST(offset);
if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE,
func_ctx->aot_inst, &offset, 1,
"shared_heap_start_off_p"))) {
aot_set_last_error("llvm build inbounds gep failed");
return false;
}
if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2(
comp_ctx->builder,
comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE,
start_off_p, "shared_heap_start_off"))) {
aot_set_last_error("llvm build load failed");
return false;
}
if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder,
func_ctx->shared_heap_base_addr_adj,
"has_shared_heap"))) {
aot_set_last_error("llvm build is not null failed");
return false;
}
return true;
fail:
return false;
}
static bool
create_cur_exception(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
{
@ -1808,6 +1877,12 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx,
goto fail;
}
/* Load shared heap, shared heap start off mem32 or mem64 */
if (comp_ctx->enable_shared_heap
&& !create_shared_heap_info(comp_ctx, func_ctx)) {
goto fail;
}
return func_ctx;
fail:
@ -2619,6 +2694,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option)
if (option->enable_gc)
comp_ctx->enable_gc = true;
if (option->enable_shared_heap)
comp_ctx->enable_shared_heap = true;
comp_ctx->opt_level = option->opt_level;
comp_ctx->size_level = option->size_level;

5
core/iwasm/compilation/aot_llvm.h
View File

@ -242,6 +242,9 @@ typedef struct AOTFuncContext {
bool mem_space_unchanged;
AOTCheckedAddrList checked_addr_list;
LLVMValueRef shared_heap_base_addr_adj;
LLVMValueRef shared_heap_start_off;
LLVMBasicBlockRef got_exception_block;
LLVMBasicBlockRef func_return_block;
LLVMValueRef exception_id_phi;
@ -467,6 +470,8 @@ typedef struct AOTCompContext {
/* Enable GC */
bool enable_gc;
bool enable_shared_heap;
uint32 opt_level;
uint32 size_level;

1
core/iwasm/include/aot_comp_option.h
View File

@ -73,6 +73,7 @@ typedef struct AOTCompOption {
bool enable_llvm_pgo;
bool enable_stack_estimation;
bool quick_invoke_c_api_import;
bool enable_shared_heap;
char *use_prof_file;
uint32_t opt_level;
uint32_t size_level;

61
core/iwasm/include/wasm_export.h
View File

@ -139,6 +139,9 @@ typedef struct wasm_section_t {
struct WASMExecEnv;
typedef struct WASMExecEnv *wasm_exec_env_t;
struct WASMSharedHeap;
typedef struct WASMSharedHeap *wasm_shared_heap_t;
/* Package Type */
typedef enum {
Wasm_Module_Bytecode = 0,
@ -329,6 +332,10 @@ typedef enum {
WASM_LOG_LEVEL_VERBOSE = 4
} log_level_t;
typedef struct SharedHeapInitArgs {
uint32_t size;
} SharedHeapInitArgs;
/**
* Initialize the WASM runtime environment, and also initialize
* the memory allocator with system allocator, which calls os_malloc
@ -2219,6 +2226,60 @@ wasm_runtime_detect_native_stack_overflow_size(wasm_exec_env_t exec_env,
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_is_underlying_binary_freeable(const wasm_module_t module);
/**
* Create a shared heap
*
* @param init_args the initialization arguments
* @return the shared heap created
*/
WASM_RUNTIME_API_EXTERN wasm_shared_heap_t
wasm_runtime_create_shared_heap(SharedHeapInitArgs *init_args);
/**
* Attach a shared heap to a module instance
*
* @param module_inst the module instance
* @param shared_heap the shared heap
* @return true if success, false if failed
*/
WASM_RUNTIME_API_EXTERN bool
wasm_runtime_attach_shared_heap(wasm_module_inst_t module_inst,
wasm_shared_heap_t shared_heap);
/**
* Detach a shared heap from a module instance
*
* @param module_inst the module instance
*/
WASM_RUNTIME_API_EXTERN void
wasm_runtime_detach_shared_heap(wasm_module_inst_t module_inst);
/**
* Allocate memory from a shared heap
*
* @param module_inst the module instance
* @param size required memory size
* @param p_native_addr native address of allocated memory
*
* @return return the allocated memory address, which re-uses part of the wasm
* address space and is in the range of [UINT32 - shared_heap_size + 1, UINT32]
* (when the wasm memory is 32-bit) or [UINT64 - shared_heap_size + 1, UINT64]
* (when the wasm memory is 64-bit). Note that it is not an absolute address.
* Return non-zero if success, zero if failed.
*/
WASM_RUNTIME_API_EXTERN uint64_t
wasm_runtime_shared_heap_malloc(wasm_module_inst_t module_inst, uint64_t size,
void **p_native_addr);
/**
* Free the memory allocated from shared heap
*
* @param module_inst the module instance
* @param ptr the offset in wasm app
*/
WASM_RUNTIME_API_EXTERN void
wasm_runtime_shared_heap_free(wasm_module_inst_t module_inst, uint64_t ptr);
#ifdef __cplusplus
}
#endif

152
core/iwasm/interpreter/wasm_interp_classic.c
View File

@ -46,6 +46,28 @@ typedef float64 CellType_F64;
#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory)
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
#if WASM_ENABLE_MULTI_MEMORY != 0
/* Only enable shared heap for the default memory */
#define is_default_memory (memidx == 0)
#else
#define is_default_memory true
#endif
#define app_addr_in_shared_heap(app_addr, bytes) \
(shared_heap && is_default_memory && (app_addr) >= shared_heap_start_off \
&& (app_addr) <= shared_heap_end_off - bytes + 1)
#define shared_heap_addr_app_to_native(app_addr, native_addr) \
native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off)
#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \
if (app_addr_in_shared_heap(app_addr, bytes)) \
shared_heap_addr_app_to_native(app_addr, native_addr); \
else
#else
#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr)
#endif
#if WASM_ENABLE_MEMORY64 == 0
#if (!defined(OS_ENABLE_HW_BOUND_CHECK) \
@ -53,6 +75,7 @@ typedef float64 CellType_F64;
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* If offset1 is in valid range, maddr must also \
be in valid range, no need to check it again. */ \
@ -64,6 +87,7 @@ typedef float64 CellType_F64;
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* App heap space is not valid space for \
bulk memory operation */ \
@ -71,18 +95,24 @@ typedef float64 CellType_F64;
else \
goto out_of_bounds; \
} while (0)
#else /* else of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
maddr = memory->memory_data + offset1; \
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
maddr = memory->memory_data + (uint32)(start); \
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0)
#endif /* end of !defined(OS_ENABLE_HW_BOUND_CHECK) || \
WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
@ -91,6 +121,7 @@ typedef float64 CellType_F64;
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
/* If memory64 is enabled, offset1, offset1 + bytes can overflow */ \
if (disable_bounds_checks \
|| (offset1 >= offset && offset1 + bytes >= offset1 \
@ -99,9 +130,11 @@ typedef float64 CellType_F64;
else \
goto out_of_bounds; \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint64)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
/* If memory64 is enabled, offset1 + bytes can overflow */ \
if (disable_bounds_checks \
|| (offset1 + bytes >= offset1 \
@ -1611,6 +1644,22 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
if (memory)
is_memory64 = memory->is_memory64;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap = module->e->shared_heap;
uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL;
#if WASM_ENABLE_MEMORY64 != 0
uint64 shared_heap_start_off =
shared_heap ? (is_memory64 ? shared_heap->start_off_mem64
: shared_heap->start_off_mem32)
: 0;
uint64 shared_heap_end_off =
shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0;
#else
uint64 shared_heap_start_off =
shared_heap ? shared_heap->start_off_mem32 : 0;
uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0;
#endif
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
#if WASM_ENABLE_MULTI_MEMORY != 0
uint32 memidx = 0;
uint32 memidx_cached = (uint32)-1;
@ -3472,8 +3521,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
str_obj = (WASMString)wasm_stringref_obj_get_value(
stringref_obj);
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
if (opcode == WASM_OP_STRING_ENCODE_WTF16) {
flag = WTF16;
@ -3640,8 +3696,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
addr = POP_I32();
stringview_wtf8_obj = POP_REF();
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
bytes_written = wasm_string_encode(
(WASMString)wasm_stringview_wtf8_obj_get_value(
@ -5674,9 +5737,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr);
#else
if ((uint64)(uint32)addr + bytes > linear_mem_size)
goto out_of_bounds;
maddr = memory->memory_data + (uint32)addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)(uint32)addr,
bytes))
shared_heap_addr_app_to_native((uint64)(uint32)addr,
maddr);
else
#endif
{
if ((uint64)(uint32)addr + bytes > linear_mem_size)
goto out_of_bounds;
maddr = memory->memory_data + (uint32)addr;
}
#endif
if (bh_bitmap_get_bit(module->e->common.data_dropped,
@ -5726,15 +5798,30 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_THREAD_MGR != 0
linear_mem_size = get_linear_mem_size();
#endif
dlen = linear_mem_size - dst;
/* dst boundary check */
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else
if ((uint64)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + dst;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)dst, len))
dlen = shared_heap_end_off - dst + 1;
#endif
dlen = linear_mem_size - dst;
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)dst, len)) {
shared_heap_addr_app_to_native((uint64)dst, mdst);
dlen = shared_heap_end_off - dst + 1;
}
else
#endif
{
if ((uint64)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + dst;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_MULTI_MEMORY != 0
/* src memidx */
@ -5750,9 +5837,16 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc);
#else
if ((uint64)src + len > linear_mem_size)
goto out_of_bounds;
msrc = memory->memory_data + src;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)src, len))
shared_heap_addr_app_to_native((uint64)src, msrc);
else
#endif
{
if ((uint64)src + len > linear_mem_size)
goto out_of_bounds;
msrc = memory->memory_data + src;
}
#endif
#if WASM_ENABLE_MEMORY64 == 0
@ -5789,9 +5883,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)(uint32)dst, len))
shared_heap_addr_app_to_native((uint64)(uint32)dst,
mdst);
else
#endif
{
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
}
#endif
memset(mdst, fill_val, len);

146
core/iwasm/interpreter/wasm_interp_fast.c
View File

@ -37,11 +37,28 @@ typedef float64 CellType_F64;
#define get_linear_mem_size() GET_LINEAR_MEMORY_SIZE(memory)
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
#define app_addr_in_shared_heap(app_addr, bytes) \
(shared_heap && (app_addr) >= shared_heap_start_off \
&& (app_addr) <= shared_heap_end_off - bytes + 1)
#define shared_heap_addr_app_to_native(app_addr, native_addr) \
native_addr = shared_heap_base_addr + ((app_addr)-shared_heap_start_off)
#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr) \
if (app_addr_in_shared_heap(app_addr, bytes)) \
shared_heap_addr_app_to_native(app_addr, native_addr); \
else
#else
#define CHECK_SHARED_HEAP_OVERFLOW(app_addr, bytes, native_addr)
#endif
#if !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* If offset1 is in valid range, maddr must also \
be in valid range, no need to check it again. */ \
@ -53,6 +70,7 @@ typedef float64 CellType_F64;
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
if (disable_bounds_checks || offset1 + bytes <= get_linear_mem_size()) \
/* App heap space is not valid space for \
bulk memory operation */ \
@ -61,15 +79,18 @@ typedef float64 CellType_F64;
goto out_of_bounds; \
} while (0)
#else
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
maddr = memory->memory_data + offset1; \
#define CHECK_MEMORY_OVERFLOW(bytes) \
do { \
uint64 offset1 = (uint64)offset + (uint64)addr; \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0)
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
maddr = memory->memory_data + (uint32)(start); \
#define CHECK_BULK_MEMORY_OVERFLOW(start, bytes, maddr) \
do { \
uint64 offset1 = (uint32)(start); \
CHECK_SHARED_HEAP_OVERFLOW(offset1, bytes, maddr) \
maddr = memory->memory_data + offset1; \
} while (0)
#endif /* !defined(OS_ENABLE_HW_BOUND_CHECK) \
|| WASM_CPU_SUPPORTS_UNALIGNED_ADDR_ACCESS == 0 */
@ -1516,6 +1537,24 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#if WASM_ENABLE_TAIL_CALL != 0 || WASM_ENABLE_GC != 0
bool is_return_call = false;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap = module->e ? module->e->shared_heap : NULL;
uint8 *shared_heap_base_addr = shared_heap ? shared_heap->base_addr : NULL;
/*
#if WASM_ENABLE_MEMORY64 != 0
uint64 shared_heap_start_off =
shared_heap ? (is_memory64 ? shared_heap->start_off_mem64
: shared_heap->start_off_mem32)
: 0;
uint64 shared_heap_end_off =
shared_heap ? (is_memory64 ? UINT64_MAX : UINT32_MAX) : 0;
#else
*/ /* TODO: uncomment the code when memory64 is enabled for fast-interp */
uint64 shared_heap_start_off =
shared_heap ? shared_heap->start_off_mem32 : 0;
uint64 shared_heap_end_off = shared_heap ? UINT32_MAX : 0;
/* #endif */
#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */
#if WASM_ENABLE_LABELS_AS_VALUES != 0
#define HANDLE_OPCODE(op) &&HANDLE_##op
@ -2831,8 +2870,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
str_obj = (WASMString)wasm_stringref_obj_get_value(
stringref_obj);
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
if (opcode == WASM_OP_STRING_ENCODE_WTF16) {
flag = WTF16;
@ -2999,8 +3045,15 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
addr = POP_I32();
stringview_wtf8_obj = POP_REF();
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)addr, 1))
shared_heap_addr_app_to_native((uint64)addr, maddr);
else
#endif
{
memory_inst = module->memories[mem_idx];
maddr = memory_inst->memory_data + addr;
}
bytes_written = wasm_string_encode(
(WASMString)wasm_stringview_wtf8_obj_get_value(
@ -4985,9 +5038,18 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(addr, bytes, maddr);
#else
if ((uint64)(uint32)addr + bytes > linear_mem_size)
goto out_of_bounds;
maddr = memory->memory_data + (uint32)addr;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)(uint32)addr,
bytes))
shared_heap_addr_app_to_native((uint64)(uint32)addr,
maddr);
else
#endif
{
if ((uint64)(uint32)addr + bytes > linear_mem_size)
goto out_of_bounds;
maddr = memory->memory_data + (uint32)addr;
}
#endif
if (bh_bitmap_get_bit(module->e->common.data_dropped,
segment)) {
@ -5020,6 +5082,7 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
{
uint32 dst, src, len;
uint8 *mdst, *msrc;
uint64 dlen;
len = POP_I32();
src = POP_I32();
@ -5029,22 +5092,43 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
linear_mem_size = get_linear_mem_size();
#endif
dlen = linear_mem_size - dst;
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(src, len, msrc);
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else
if ((uint64)(uint32)src + len > linear_mem_size)
goto out_of_bounds;
msrc = memory->memory_data + (uint32)src;
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)dst, len))
dlen = shared_heap_end_off - dst + 1;
#endif
#else /* else of OS_ENABLE_HW_BOUND_CHECK */
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)src, len))
shared_heap_addr_app_to_native((uint64)src, msrc);
else
#endif
{
if ((uint64)(uint32)src + len > linear_mem_size)
goto out_of_bounds;
msrc = memory->memory_data + (uint32)src;
}
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)dst, len)) {
shared_heap_addr_app_to_native((uint64)dst, mdst);
dlen = shared_heap_end_off - dst + 1;
}
else
#endif
{
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
}
#endif /* end of OS_ENABLE_HW_BOUND_CHECK */
/* allowing the destination and source to overlap */
bh_memmove_s(mdst, (uint32)(linear_mem_size - dst),
msrc, len);
bh_memmove_s(mdst, (uint32)dlen, msrc, len);
break;
}
case WASM_OP_MEMORY_FILL:
@ -5063,9 +5147,17 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module,
#ifndef OS_ENABLE_HW_BOUND_CHECK
CHECK_BULK_MEMORY_OVERFLOW(dst, len, mdst);
#else
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
#if WASM_ENABLE_SHARED_HEAP != 0
if (app_addr_in_shared_heap((uint64)(uint32)dst, len))
shared_heap_addr_app_to_native((uint64)(uint32)dst,
mdst);
else
#endif
{
if ((uint64)(uint32)dst + len > linear_mem_size)
goto out_of_bounds;
mdst = memory->memory_data + (uint32)dst;
}
#endif
memset(mdst, fill_val, len);

3
core/iwasm/interpreter/wasm_loader.c
View File

@ -5379,6 +5379,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
option.enable_memory_profiling = true;
option.enable_stack_estimation = true;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
option.enable_shared_heap = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {

3
core/iwasm/interpreter/wasm_mini_loader.c
View File

@ -2216,6 +2216,9 @@ init_llvm_jit_functions_stage1(WASMModule *module, char *error_buf,
option.enable_memory_profiling = true;
option.enable_stack_estimation = true;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
option.enable_shared_heap = true;
#endif
module->comp_ctx = aot_create_comp_context(module->comp_data, &option);
if (!module->comp_ctx) {

8
core/iwasm/interpreter/wasm_runtime.c
View File

@ -2798,6 +2798,14 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent,
}
}
#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0
#if UINTPTR_MAX == UINT64_MAX
module_inst->e->shared_heap_start_off.u64 = UINT64_MAX;
#else
module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX;
#endif
#endif
#if WASM_ENABLE_GC != 0
/* Initialize the table data with init expr */
for (i = 0; i < module->table_count; i++) {

22
core/iwasm/interpreter/wasm_runtime.h
View File

@ -92,6 +92,15 @@ typedef union {
uint32 u32[2];
} MemBound;
typedef struct WASMSharedHeap {
struct WASMSharedHeap *next;
void *heap_handle;
uint8 *base_addr;
uint64 size;
uint64 start_off_mem64;
uint64 start_off_mem32;
} WASMSharedHeap;
struct WASMMemoryInstance {
/* Module type */
uint32 module_type;
@ -354,6 +363,19 @@ typedef struct WASMModuleInstanceExtra {
uint32 max_aux_stack_used;
#endif
#if WASM_ENABLE_SHARED_HEAP != 0
WASMSharedHeap *shared_heap;
#if WASM_ENABLE_JIT != 0
/*
* Adjusted shared heap based addr to simple the calculation
* in the aot code. The value is:
* shared_heap->base_addr - shared_heap->start_off
*/
uint8 *shared_heap_base_addr_adj;
MemBound shared_heap_start_off;
#endif
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0 \
|| (WASM_ENABLE_FAST_JIT != 0 && WASM_ENABLE_JIT != 0 \
&& WASM_ENABLE_LAZY_JIT != 0)

8
core/iwasm/libraries/shared-heap/shared_heap.cmake Normal file
View File

@ -0,0 +1,8 @@
# Copyright (C) 2024 Xiaomi Corporation. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
set (LIB_SHARED_HEAP ${CMAKE_CURRENT_LIST_DIR})
add_definitions (-DWASM_ENABLE_SHARED_HEAP=1)
include_directories(${LIB_SHARED_HEAP_DIR})
file (GLOB source_all ${LIB_SHARED_HEAP}/*.c)
set (LIB_SHARED_HEAP_SOURCE ${source_all})

57
core/iwasm/libraries/shared-heap/shared_heap_wrapper.c Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright (C) 2024 Xiaomi Corporation. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*/
#include "bh_common.h"
#include "bh_log.h"
#include "wasm_export.h"
#include "../interpreter/wasm.h"
#include "../common/wasm_runtime_common.h"
/* clang-format off */
#define validate_native_addr(addr, size) \
wasm_runtime_validate_native_addr(module_inst, addr, size)
#define module_shared_malloc(size, p_native_addr) \
wasm_runtime_shared_heap_malloc(module_inst, size, p_native_addr)
#define module_shared_free(offset) \
wasm_runtime_shared_heap_free(module_inst, offset)
/* clang-format on */
static uint32
shared_heap_malloc_wrapper(wasm_exec_env_t exec_env, uint32 size)
{
wasm_module_inst_t module_inst = get_module_inst(exec_env);
return (uint32)module_shared_malloc((uint64)size, NULL);
}
static void
shared_heap_free_wrapper(wasm_exec_env_t exec_env, void *ptr)
{
wasm_module_inst_t module_inst = get_module_inst(exec_env);
if (!validate_native_addr(ptr, (uint64)sizeof(uintptr_t))) {
LOG_WARNING("Invalid app address");
return;
}
module_shared_free(addr_native_to_app(ptr));
}
/* clang-format off */
#define REG_NATIVE_FUNC(func_name, signature) \
{ #func_name, func_name##_wrapper, signature, NULL }
/* clang-format on */
static NativeSymbol native_symbols_shared_heap[] = {
REG_NATIVE_FUNC(shared_heap_malloc, "(i)i"),
REG_NATIVE_FUNC(shared_heap_free, "(*)"),
};
uint32
get_lib_shared_heap_export_apis(NativeSymbol **p_shared_heap_apis)
{
*p_shared_heap_apis = native_symbols_shared_heap;
return sizeof(native_symbols_shared_heap) / sizeof(NativeSymbol);
}

76
core/iwasm/libraries/thread-mgr/thread_manager.c
View File

@ -1402,6 +1402,82 @@ wasm_cluster_spread_custom_data(WASMModuleInstanceCommon *module_inst,
}
}
#if WASM_ENABLE_SHARED_HEAP != 0
static void
attach_shared_heap_visitor(void *node, void *heap)
{
WASMExecEnv *curr_exec_env = (WASMExecEnv *)node;
WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env);
wasm_runtime_attach_shared_heap_internal(module_inst, heap);
}
static void
detach_shared_heap_visitor(void *node, void *heap)
{
WASMExecEnv *curr_exec_env = (WASMExecEnv *)node;
WASMModuleInstanceCommon *module_inst = get_module_inst(curr_exec_env);
(void)heap;
wasm_runtime_detach_shared_heap_internal(module_inst);
}
bool
wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *heap)
{
WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst);
if (exec_env == NULL) {
/* Maybe threads have not been started yet. */
return wasm_runtime_attach_shared_heap_internal(module_inst, heap);
}
else {
WASMCluster *cluster;
cluster = wasm_exec_env_get_cluster(exec_env);
bh_assert(cluster);
os_mutex_lock(&cluster->lock);
/* Try attaching shared heap to this module instance first
to ensure that we can attach it to all other instances. */
if (!wasm_runtime_attach_shared_heap_internal(module_inst, heap)) {
os_mutex_unlock(&cluster->lock);
return false;
}
/* Detach the shared heap so it can be attached again. */
wasm_runtime_detach_shared_heap_internal(module_inst);
traverse_list(&cluster->exec_env_list, attach_shared_heap_visitor,
heap);
os_mutex_unlock(&cluster->lock);
}
return true;
}
void
wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst)
{
WASMExecEnv *exec_env = wasm_clusters_search_exec_env(module_inst);
if (exec_env == NULL) {
/* Maybe threads have not been started yet. */
wasm_runtime_detach_shared_heap_internal(module_inst);
}
else {
WASMCluster *cluster;
cluster = wasm_exec_env_get_cluster(exec_env);
bh_assert(cluster);
os_mutex_lock(&cluster->lock);
traverse_list(&cluster->exec_env_list, detach_shared_heap_visitor,
NULL);
os_mutex_unlock(&cluster->lock);
}
}
#endif
#if WASM_ENABLE_MODULE_INST_CONTEXT != 0
struct inst_set_context_data {
void *key;

12
core/iwasm/libraries/thread-mgr/thread_manager.h
View File

@ -11,6 +11,9 @@
#include "wasm_export.h"
#include "../interpreter/wasm.h"
#include "../common/wasm_runtime_common.h"
#if WASM_ENABLE_SHARED_HEAP != 0
#include "../common/wasm_memory.h"
#endif
#ifdef __cplusplus
extern "C" {
@ -167,6 +170,15 @@ wasm_cluster_set_context(WASMModuleInstanceCommon *module_inst, void *key,
bool
wasm_cluster_is_thread_terminated(WASMExecEnv *exec_env);
#if WASM_ENABLE_SHARED_HEAP != 0
bool
wasm_cluster_attach_shared_heap(WASMModuleInstanceCommon *module_inst,
WASMSharedHeap *heap);
void
wasm_cluster_detach_shared_heap(WASMModuleInstanceCommon *module_inst);
#endif
#if WASM_ENABLE_DEBUG_INTERP != 0
#define WAMR_SIG_TRAP (5)
#define WAMR_SIG_STOP (19)